@discord/intl-loader-core 0.9.0-rc.1 → 0.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@discord/intl-loader-core",
-  "version": "0.9.0-rc.1",
+  "version": "0.9.0",
   "license": "MIT",
   "description": "Core utilities for writing loaders and transformers using @discord/intl",
   "author": "Jon Egeland",
@@ -9,6 +9,7 @@
   "files": [
     "index.js",
     "types.d.ts",
+    "src",
     "types"
   ],
   "exports": {
@@ -25,7 +26,7 @@
     "chokidar": "^3.6.0",
     "debug": "^4.3.6",
     "fast-glob": "^3.3.2",
-    "@discord/intl-message-database": "0.9.0-rc.1"
+    "@discord/intl-message-database": "0.9.0"
   },
   "devDependencies": {
     "@types/debug": "^4.1.12",

package/src/database.js

@@ -0,0 +1,9 @@
+const { IntlMessagesDatabase } = require('@discord/intl-message-database');
+
+/**
+ * A shared message database instance that's used and shared across all parts
+ * of the loader and plugin together.
+ */
+const database = new IntlMessagesDatabase();
+
+module.exports = { database };

package/src/processing.js

@@ -0,0 +1,195 @@
+const path = require('node:path');
+
+const debug = require('debug')('intl:loader-core');
+const { IntlCompiledMessageFormat } = require('@discord/intl-message-database');
+
+const { database } = require('./database');
+const { findAllTranslationFiles, getLocaleFromTranslationsFileName } = require('./util');
+
+/**
+ * @typedef {{
+ *   format?: IntlCompiledMessageFormat,
+ *   bundleSecrets?: boolean,
+ * }} IntlPrecompileOptions
+ */
+
+/**
+ * @param {string} sourcePath
+ * @param {import('@discord/intl-message-database').IntlSourceFile} sourceFile
+ */
+function debugSourceFile(sourcePath, sourceFile) {
+  debug(
+    `[${sourcePath}] Parsed messages file: type=${sourceFile.type}, locale=${sourceFile.locale}, messageCount=${sourceFile.messageKeys.length}, meta=${JSON.stringify(sourceFile.meta)}`,
+  );
+}
+
+/**
+ *
+ * @param {string} sourcePath Path of the source file being processed, for debug logging
+ * @param {import('@discord/intl-message-database').IntlSourceFile} sourceFile SourceFile object from the database used to find translations.
+ * @param {string} translationsPath Fully-resolved path to the directory for translations.
+ * @returns {Record<string, string>}
+ */
+function buildTranslationsLocaleMap(sourcePath, sourceFile, translationsPath) {
+  if (sourceFile.meta.translate === false) {
+    debug(`[${sourcePath}] translate is set to false, no locale map is needed`);
+    return {};
+  }
+  const map = findAllTranslationFiles(translationsPath);
+  if (map instanceof Error) {
+    debug(`[${sourcePath}] Failed to build locale map: [${map.name}] ${map.message}`);
+    return {};
+  }
+  return map;
+}
+
+/**
+ * @param {string} sourcePath
+ * @param {string=} sourceContent
+ * @param {{
+ *   processTranslations?: boolean,
+ *   locale?: string
+ * }=} options
+ * @returns {import('../types').ProcessDefinitionsResult}
+ */
+function processDefinitionsFile(sourcePath, sourceContent, options = {}) {
+  const {
+    processTranslations = false,
+    // TODO: Make this more configurable/automatically determined.
+    locale = 'en-US',
+  } = options;
+  debug(`[${sourcePath}] Processing definitions with locale "${locale}"`);
+
+  if (sourceContent != null) {
+    database.processDefinitionsFileContent(sourcePath, sourceContent);
+  } else {
+    database.processDefinitionsFile(sourcePath);
+  }
+
+  const sourceFile = database.getSourceFile(sourcePath);
+  debugSourceFile(sourcePath, sourceFile);
+  if (sourceFile.type !== 'definition') {
+    throw new Error(
+      `Expected ${sourcePath} to be a message definitions file, but it resulted in ${sourceFile.type} instead.`,
+    );
+  }
+
+  const messageKeys = database.getSourceFileKeyMap(sourcePath);
+  const translationsPath = path.resolve(path.dirname(sourcePath), sourceFile.meta.translationsPath);
+  const translationsLocaleMap = buildTranslationsLocaleMap(
+    sourcePath,
+    sourceFile,
+    translationsPath,
+  );
+
+  if (processTranslations) {
+    database.processAllTranslationFiles(translationsLocaleMap);
+  }
+
+  return {
+    sourceFile,
+    locale,
+    messageKeys,
+    translationsPath,
+    translationsLocaleMap,
+  };
+}
+
+/**
+ *
+ * @param {string} sourcePath
+ * @param {string=} sourceContent
+ * @param {{
+ *   locale?: string,
+ * }=} options
+ * @returns {import('../types').ProcessTranslationsResult}
+ */
+function processTranslationsFile(sourcePath, sourceContent, options = {}) {
+  const { locale = getLocaleFromTranslationsFileName(sourcePath) } = options;
+  if (sourceContent) {
+    database.processTranslationFileContent(sourcePath, locale, sourceContent);
+  } else {
+    database.processTranslationFile(sourcePath, locale);
+  }
+
+  const sourceFile = database.getSourceFile(sourcePath);
+  debugSourceFile(sourcePath, sourceFile);
+  if (sourceFile.type !== 'translation') {
+    throw new Error(
+      `Expected ${sourcePath} to be a message translations file, but it resulted in ${sourceFile.type} instead.`,
+    );
+  }
+
+  return {
+    sourceFile,
+    locale,
+    messageKeys: database.getSourceFileKeyMap(sourcePath),
+  };
+}
+
+/**
+ * Precompile the messages defined in the given `sourcePath` using the value of the translation for
+ * that message in the given `locale`. `format` specifies which serialization format the result
+ * will be written in.
+ *
+ * By default, the compiled content will be returned as a Buffer containing the serialized string,
+ * but if `outputFile` is given then the content will be written directly to the file and the
+ * function becomes `void`.
+ *
+ * Compiling automatically handles filtering out messages based on the meta information like
+ * `translate`, `secret`, and `bundleSecrets`, to ensure that all consumers apply these values
+ * accurately and consistently.
+ *
+ * @param {string} sourcePath
+ * @param {string} locale
+ * @param {string=} outputFile
+ * @param {IntlPrecompileOptions} [options]
+ *
+ * @returns {Buffer | void}
+ */
+function precompileFileForLocale(sourcePath, locale, outputFile, options = {}) {
+  return outputFile != null
+    ? database.precompile(sourcePath, locale, outputFile, options)
+    : database.precompileToBuffer(sourcePath, locale, options);
+}
+
+/**
+ * Generate a `.d.ts` file containing TypeScript type definitions for all of the messages defined in
+ * `sourcePath`. This method does not process `sourcePath` at all, meaning it expects the database
+ * to already know about the source, as well as all of the related translations to create an
+ * accurate typescript definition for each message.
+ *
+ * If not given, `outputFile` will default to the same path as `sourcePath`, with the last extension
+ * replaced by `.d.ts`. For example, a file like `SomeMessages.Other.messages.js` would become
+ * `SomeMessages.Other.messages.d.ts`.
+ *
+ * If `allowNullability` is set, the generated types for variables within messages will allow
+ * `null` and `undefined` for most value types, as well as looser restrictions on typing, such as
+ * allowing `string | number` for number variables.
+ *
+ * Returns `true` if the types were successfully generated, or `false` otherwise, such as if the
+ * source file is not already in the database.
+ *
+ * @param {string} sourcePath
+ * @param {string=} outputFile
+ * @param {boolean=} allowNullability
+ * @returns {boolean}
+ */
+function generateTypeDefinitions(sourcePath, outputFile, allowNullability = false) {
+  const paths = database.getAllSourceFilePaths();
+  if (!paths.includes(sourcePath)) return false;
+
+  database.generateTypes(
+    sourcePath,
+    outputFile ?? sourcePath.replace(/\.[^.]+$/, '.d.ts'),
+    allowNullability,
+  );
+  return true;
+}
+
+module.exports = {
+  generateTypeDefinitions,
+  precompileFileForLocale,
+  processDefinitionsFile,
+  processTranslationsFile,
+};

package/src/transformer.js

@@ -0,0 +1,156 @@
+/**
+ * Common class for parsing and transforming the content of a messages
+ * definition file (e.g., "SomeFeature.messages.js") into a production-ready
+ * version, with message keys obfuscated, loading harnesses configured, and
+ * more.
+ *
+ * This transformation is intended to be used alongside the _consumer_
+ * transforms implemented as SWC and Babel plugins, which transform the
+ * callsites for messages into matching formats. Consider this input example:
+ *
+ * ```typescript
+ * // SomeModule.messages.js
+ * import {defineMessages} from '@discord/intl';
+ *
+ * export default defineMessages({
+ *   THIS_IS_A_MESSAGE: 'it has some content with {values}',
+ * });
+ *
+ * // SomeConsumer.tsx
+ * import someModuleMessages from 'SomeModule.messages.js';
+ * i18n.format(someModuleMessages.THIS_IS_A_MESSAGE, {values: "I'm a value!"});
+ * ```
+ *
+ * This transformer will only handle `SomeModule.messages.js`, and will output
+ * something like:
+ *
+ * ```typescript
+ * const {i18n} = require('@discord/intl');
+ * const _keys = ["a9fn23"];
+ * const _locales = {"en-US": () => require('./messages/en-US.messages.json')};
+ * export const messagesLoader = createLoader(_keys, _locales);
+ * export default messagesLoader.getBinds();
+ * ```
+ *
+ * Notice how the message keys have been hashed into short keys, and the
+ * dynamic imports for each locale's data have been inserted automatically. In
+ * this example, the only locale is the source locale.
+ *
+ * The SWC and Babel transformers then take care of the second file,
+ * transforming the usage to use the hashed keys:
+ *
+ * ```typescript
+ * import someModuleMessages from 'SomeModule.messages.js';
+ * i18n.format(someModuleMessages["a9fn23"], {values: "i'm a value!"});
+ * ```
+ *
+ * The transformed file also contains a named export for `messagesLoader`,
+ * which consumers can use to query and update the loading state for the
+ * messages managed by that loader, including waiting for a locale to be
+ * loaded, kicking off new loads, and more:
+ *
+ * ```typescript
+ * import {messagesLoader} from 'SomeModule.messages.js';
+ * // Wait for the loader to be initialized with default messages
+ * await messagesLoader.waitForDefaultLocaleLoaded();
+ * // Wait for a specific locale to load, starting the load if it
+ * // is not yet in progress.
+ * await messagesLoader.waitForLocaleLoaded('fr');
+ * // In hot-reloading environments, use the second `requireCurrent`
+ * // parameter to wait for the latest data, even if a value already
+ * // exists.
+ * const loaded = messagesLoader.isLocaleLoaded('fr', true);
+ * ```
+ */
+class MessageDefinitionsTransformer {
+  /**
+   * @param {import('../types.d.ts').MessageDefinitionsTransformerOptions} options
+   */
+  constructor(options) {
+    this.options = options;
+    this.loaderName = 'messagesLoader';
+  }
+
+  /**
+   * Returns a compiled string for an object that maps locale names to a
+   * dynamic require function for that locale, based on the supported locales
+   * that were determined for this file. The shape ends up as:
+   *
+   * ```typescript
+   * {
+   *   "en-US": () => import("path/to/en-US.json"),
+   * }
+   * ```
+   *
+   * @returns {string}
+   */
+  getLocaleRequireMap() {
+    const localeProperties = [];
+    for (const [locale, importPath] of Object.entries(this.options.localeMap)) {
+      // This assumes that the author has specified `importPath`
+      // as a properly-resolvable path for the bundler, which we can't easily
+      // enforce, unfortunately.
+      localeProperties.push(`"${locale}": () => ${this.options.getTranslationImport(importPath)}`);
+    }
+
+    return `{${localeProperties.join(',')}}`;
+  }
+
+  /**
+   * Return a map of key hashes to their original values, as well as a plain-text map of locales
+   * to the file names that they import from.
+   *
+   * @returns {string[]}
+   */
+  debugModeSetup() {
+    if (!this.options.debug) return [];
+
+    return [
+      `${this.loaderName}.withDebugValues(${JSON.stringify(this.options.messageKeys)}, ${JSON.stringify(this.options.localeMap)})`,
+    ];
+  }
+
+  /**
+   * Return the lines to export fields from this module, as determined by the `exportMode` on this
+   * transformer.
+   */
+  exportFields() {
+    switch (this.options.exportMode ?? 'esm') {
+      case 'esm':
+        return [`export {${this.loaderName}};`, `export default binds;`];
+      case 'commonjs':
+        return [`module.exports = { messagesLoader: ${this.loaderName}, default: binds };`];
+      case 'transpiledEsModule':
+        return [
+          `Object.defineProperty(exports, "__esModule", { value: true });`,
+          `exports["messageLoader"] = ${this.loaderName};`,
+          `exports["default"] = binds;`,
+        ];
+    }
+  }
+
+  /**
+   * Returns the reduced, transformed output for this file. Currently not
+   * configurable, but could be told to include default messages or preserve
+   * information as necessary.
+   *
+   * @returns {string}
+   */
+  getOutput() {
+    return [
+      this.options.getPrelude?.() ?? '// No additional prelude was configured.',
+      `const {createLoader} = require('@discord/intl');`,
+      `const _keys = ${JSON.stringify(this.options.messageKeys)};`,
+      `const _locales = ${this.getLocaleRequireMap()};`,
+      `const _defaultLocale = ${JSON.stringify(this.options.defaultLocale)};`,
+      `const ${this.loaderName} = createLoader(_keys, _locales, _defaultLocale);`,
+      ...this.debugModeSetup(),
+      `const binds = ${this.loaderName}.getBinds();`,
+      ...this.exportFields(),
+    ].join('\n');
+  }
+}
+
+module.exports = {
+  MessageDefinitionsTransformer,
+};

package/src/util.js

@@ -0,0 +1,55 @@
+const path = require('node:path');
+const fs = require('node:fs');
+const { isMessageTranslationsFile } = require('@discord/intl-message-database');
+
+const IGNORED_MESSAGE_FILE_PATTERNS = [/.*\.compiled.messages\..*/];
+
+/**
+ * Return the presumed locale for a translations file from it's name. The convention follows the
+ * format: `some/path/to/<locale>.messages.jsona`, so the locale is determined by taking the content
+ * of the basename up until the first `.`.
+ *
+ * @param {string} fileName
+ * @returns {string}
+ */
+function getLocaleFromTranslationsFileName(fileName) {
+  return path.basename(fileName).split('.')[0];
+}
+
+/**
+ * Scan the given `translationsPath` to discover all translation files that exist, returning them
+ * as a map from locale name to the path for importing.
+ *
+ * @param {string} translationsPath
+ * @returns {Record<string, string> | Error}
+ */
+function findAllTranslationFiles(translationsPath) {
+  /** @type {Record<string, string>} */
+  const localeMap = {};
+
+  try {
+    const translationFiles = fs.readdirSync(translationsPath, { encoding: 'utf-8' });
+    for (const foundFile of translationFiles) {
+      const filePath = path.join(translationsPath, foundFile);
+      // Only include translation files, not definitions files.
+      if (!isMessageTranslationsFile(filePath)) continue;
+      // Some files are excluded, like pre-compiled artifacts.
+      if (IGNORED_MESSAGE_FILE_PATTERNS.some((pattern) => pattern.test(filePath))) continue;
+
+      const locale = getLocaleFromTranslationsFileName(filePath);
+      localeMap[locale] = filePath;
+    }
+  } catch (e) {
+    return new Error(
+      `The translations directory ${translationsPath} was not found. No translations will be loaded for these messages`,
+    );
+  }
+
+  return localeMap;
+}
+
+module.exports = {
+  IGNORED_MESSAGE_FILE_PATTERNS,
+  findAllTranslationFiles,
+  getLocaleFromTranslationsFileName,
+};

package/src/watcher.js

@@ -0,0 +1,101 @@
+const path = require('node:path');
+
+const chokidar = require('chokidar');
+const fg = require('fast-glob');
+const debug = require('debug')('intl:loader-core:watcher');
+const {
+  isMessageDefinitionsFile,
+  IntlCompiledMessageFormat,
+} = require('@discord/intl-message-database');
+
+const { database } = require('./database');
+const { processDefinitionsFile, precompileFileForLocale } = require('./processing');
+
+const ALWAYS_IGNORE_PATTERNS = [
+  // Ignore our own compiled message files, even though they shouldn't have a matching extension.
+  '*.compiled.messages.*',
+];
+// TODO: This should come from the database extension? Or Utilities? Unsure, but the extensions
+// should have some centralized location in general
+const MESSAGE_DEFINITION_FILE_PATTERNS = ['**/*.messages.js'];
+const DEFAULT_LOCALE = 'en-US';
+
+/**
+ * @param {string} filePath
+ * @param {string} assetExtension
+ * @param {import('./processing').IntlPrecompileOptions} [options]
+ */
+function processFile(filePath, assetExtension, options = {}) {
+  const { format = IntlCompiledMessageFormat.KeylessJson, bundleSecrets = false } = options;
+  debug(`Processing file: ${filePath}`);
+  if (!isMessageDefinitionsFile(filePath)) {
+    debug(`${filePath} is not a definitions file. Skipping processing`);
+    return;
+  }
+
+  try {
+    // Convert the file name from `.messages.js` to `.compiled.messages.jsona` for output.
+    const outputPath = filePath.replace(/\.messages\.js$/, `.compiled.messages.${assetExtension}`);
+    const result = processDefinitionsFile(filePath);
+    precompileFileForLocale(filePath, result.locale, undefined, {
+      format,
+      bundleSecrets,
+    });
+
+    database.processDefinitionsFile(filePath);
+    database.precompile(filePath, DEFAULT_LOCALE, outputPath, {
+      format: IntlCompiledMessageFormat.KeylessJson,
+      bundleSecrets,
+    });
+    debug(`Wrote definitions to: ${outputPath}`);
+  } catch (e) {
+    debug('[INTL Error] Failed to compile messages');
+    console.error(e);
+  }
+}
+
+/**
+ * @param {string[]} watchedFolders
+ * @param {{
+ *  watch?: boolean,
+ *  ignore?: string[],
+ *  assetExtension?: string
+ *  precompileOptions?: import('./processing').IntlPrecompileOptions,
+ * }} options
+ */
+async function compileIntlMessageFiles(
+  watchedFolders,
+  { watch = true, ignore = [], assetExtension = 'json', precompileOptions = {} } = {},
+) {
+  const ignoredPatterns = ignore.concat(ALWAYS_IGNORE_PATTERNS);
+  const globs = watchedFolders.flatMap((folder) =>
+    MESSAGE_DEFINITION_FILE_PATTERNS.map((pattern) => path.join(folder, pattern)),
+  );
+  debug(`Configured message file patterns:\n- ${globs.join('\n- ')}`);
+
+  // Perform one initial scan and compilation to ensure all files exist before Metro might try to
+  // resolve them.
+  debug('Scanning for initial messages files');
+  for await (const filePath of fg.stream(globs, {
+    ignore: ignoredPatterns,
+    absolute: true,
+    onlyFiles: true,
+  })) {
+    processFile(filePath.toString(), assetExtension, precompileOptions);
+  }
+  debug('Initial message scan completed.');
+
+  if (watch) {
+    debug(`Setting up file watching for configured paths`);
+    chokidar
+      .watch(globs, { ignored: ignoredPatterns, ignoreInitial: true })
+      .on('all', (event, filePath) => {
+        debug(`Got event ${event} for ${filePath}`);
+        processFile(filePath, assetExtension, precompileOptions);
+      });
+  } else {
+    debug('Not watching files because `watch` option was false');
+  }
+}
+
+module.exports = { compileIntlMessageFiles, ALWAYS_IGNORE_PATTERNS };