@discord/intl-loader-core 0.0.3-canary.331f0a6

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Discord, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,10 @@
+# @discord/intl-loader-core
+
+This package acts as a core set of utilities and processes that loaders and transformers can rely on to implement
+translation discovery, compilation, and management in a consistent way. It implements a core transformer to compile a
+source file into a loader runtime for `@discord/intl`, as well as functions for scanning the file system for
+translations, watching for changes, emitting typescript type definitions, and more.
+
+For the most part, consuming packages should never have to interact with the message database directly when using this
+package, which allows changes in the native extension to be masked and swapped out as needed. However, a `database`
+instance is exposed for cases where additional functionality is needed.

package/index.js

@@ -0,0 +1,35 @@
+const {
+  hashMessageKey,
+  isMessageDefinitionsFile,
+  isMessageTranslationsFile,
+  IntlCompiledMessageFormat,
+} = require('@discord/intl-message-database');
+
+const { database } = require('./src/database');
+const {
+  generateTypeDefinitions,
+  processDefinitionsFile,
+  processTranslationsFile,
+  precompileFileForLocale,
+} = require('./src/processing');
+const { MessageDefinitionsTransformer } = require('./src/transformer');
+const { findAllTranslationFiles, getLocaleFromTranslationsFileName } = require('./src/util');
+const watcher = require('./src/watcher');
+
+module.exports = {
+  // @ts-expect-error This is a const enum, which TypeScript doesn't like letting you export, even
+  // though it's a tangible object that can be accessed just fine from normal JS.
+  IntlCompiledMessageFormat,
+  MessageDefinitionsTransformer,
+  database,
+  findAllTranslationFiles,
+  getLocaleFromTranslationsFileName,
+  generateTypeDefinitions,
+  hashMessageKey,
+  isMessageDefinitionsFile,
+  isMessageTranslationsFile,
+  processDefinitionsFile,
+  processTranslationsFile,
+  precompileFileForLocale,
+  watcher,
+};

package/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@discord/intl-loader-core",
+  "version": "0.0.3-canary.331f0a6",
+  "license": "MIT",
+  "description": "Core utilities for writing loaders and transformers using @discord/intl",
+  "author": "Jon Egeland",
+  "main": "index.js",
+  "files": [
+    "src"
+  ],
+  "exports": {
+    ".": "./index.js"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/discord/discord-intl"
+  },
+  "dependencies": {
+    "chokidar": "^3.6.0",
+    "debug": "^4.3.6",
+    "fast-glob": "^3.3.2",
+    "@discord/intl-message-database": "0.0.3"
+  },
+  "devDependencies": {
+    "@types/debug": "^4.1.12"
+  }
+}

package/src/database.js

@@ -0,0 +1,9 @@
+const {IntlMessagesDatabase} = require('@discord/intl-message-database');
+
+/**
+ * A shared message database instance that's used and shared across all parts
+ * of the loader and plugin together.
+ */
+const database = new IntlMessagesDatabase();
+
+module.exports = {database};

package/src/processing.js

@@ -0,0 +1,140 @@
+const path = require('node:path');
+
+const { IntlCompiledMessageFormat } = require('@discord/intl-message-database');
+
+const { database } = require('./database');
+const { findAllTranslationFiles, getLocaleFromTranslationsFileName } = require('./util');
+
+/**
+ * @param {string} sourcePath
+ * @param {string=} sourceContent
+ * @param {{
+ *   processTranslations?: boolean,
+ *   locale?: string
+ * }=} options
+ * @returns {import('./types').ProcessDefinitionsResult}
+ */
+function processDefinitionsFile(sourcePath, sourceContent, options = {}) {
+  const {
+    processTranslations = false,
+    // TODO: Make this more configurable/automatically determined.
+    locale = 'en-US',
+  } = options;
+  if (sourceContent != null) {
+    database.processDefinitionsFileContent(sourcePath, sourceContent);
+  } else {
+    database.processDefinitionsFile(sourcePath);
+  }
+
+  const sourceFile = database.getSourceFile(sourcePath);
+  if (sourceFile.type !== 'definition') {
+    throw new Error(
+      `Expected ${sourcePath} to be a message definitions file, but it resulted in ${sourceFile.type} instead.`,
+    );
+  }
+
+  const hashedMessageKeys = database.getSourceFileHashedKeys(sourcePath);
+  const translationsPath = path.resolve(path.dirname(sourcePath), sourceFile.meta.translationsPath);
+  const translationsLocaleMap = findAllTranslationFiles(translationsPath);
+
+  if (processTranslations) {
+    database.processAllTranslationFiles(translationsLocaleMap);
+  }
+
+  return {
+    sourceFile,
+    locale,
+    hashedMessageKeys,
+    translationsPath,
+    translationsLocaleMap,
+  };
+}
+
+/**
+ *
+ * @param {string} sourcePath
+ * @param {string=} sourceContent
+ * @param {{
+ *   locale?: string,
+ *   outputFile?: string
+ * }=} options
+ * @returns {import('./types').ProcessTranslationsResult}
+ */
+function processTranslationsFile(sourcePath, sourceContent, options = {}) {
+  const { locale = getLocaleFromTranslationsFileName(sourcePath), outputFile } = options;
+  if (sourceContent) {
+    database.processTranslationFileContent(sourcePath, locale, sourceContent);
+  } else {
+    database.processTranslationFile(sourcePath, locale);
+  }
+
+  const sourceFile = database.getSourceFile(sourcePath);
+  if (sourceFile.type !== 'translation') {
+    throw new Error(
+      `Expected ${sourcePath} to be a message translations file, but it resulted in ${sourceFile.type} instead.`,
+    );
+  }
+
+  return {
+    sourceFile,
+    locale,
+    hashedMessageKeys: database.getSourceFileHashedKeys(sourcePath),
+  };
+}
+
+/**
+ * Precompile the messages defined in the given `sourcePath` using the value of the translation for
+ * that message in the given `locale`. `format` specifies which serialization format the result
+ * will be written in.
+ *
+ * By default, the compiled content will be returned as a Buffer containing the serialized string,
+ * but if `outputFile` is given then the content will be written directly to the file and the
+ * function becomes `void`.
+ *
+ * @param {string} sourcePath
+ * @param {string} locale
+ * @param {{
+ *   format?: IntlCompiledMessageFormat,
+ *   outputFile?: string
+ * }=} options
+ *
+ * @returns {Buffer | void}
+ */
+function precompileFileForLocale(sourcePath, locale, options = {}) {
+  const { format = IntlCompiledMessageFormat.Json, outputFile } = options;
+  return outputFile != null
+    ? database.precompile(sourcePath, locale, outputFile, format)
+    : database.precompileToBuffer(sourcePath, locale, format);
+}
+
+/**
+ * Generate a `.d.ts` file containing TypeScript type definitions for all of the messages defined in
+ * `sourcePath`. This method does not process `sourcePath` at all, meaning it expects the database
+ * to already know about the source, as well as all of the related translations to create an
+ * accurate typescript definition for each message.
+ *
+ * If not given, `outputFile` will default to the same path as `sourcePath`, with the last extension
+ * replaced by `.d.ts`. For example, a file like `SomeMessages.Other.messages.js` would become
+ * `SomeMessages.Other.messages.d.ts`.
+ *
+ * Returns `true` if the types were successfully generated, or `false` otherwise, such as if the
+ * source file is not already in the database.
+ *
+ * @param {string} sourcePath
+ * @param {string=} outputFile
+ * @returns {boolean}
+ */
+function generateTypeDefinitions(sourcePath, outputFile) {
+  const paths = database.getAllSourceFilePaths();
+  if (!paths.includes(sourcePath)) return false;
+
+  database.generateTypes(sourcePath, outputFile ?? sourcePath.replace(/\.[^.]+/, '.d.ts'));
+  return true;
+}
+
+module.exports = {
+  generateTypeDefinitions,
+  precompileFileForLocale,
+  processDefinitionsFile,
+  processTranslationsFile,
+};

package/src/transformer.js

@@ -0,0 +1,101 @@
+/**
+ * Common class for parsing and transforming the content of a messages
+ * definition file (e.g., "SomeFeature.messages.js") into a production-ready
+ * version, with message keys obfuscated, loading harnesses configured, and
+ * more.
+ *
+ * This transformation is intended to be used alongside the _consumer_
+ * transforms implemented as SWC and Babel plugins, which transform the
+ * callsites for messages into matching formats. Consider this input example:
+ *
+ * ```typescript
+ * // SomeModule.messages.js
+ * import {defineMessages} from '@discord/intl';
+ *
+ * export default defineMessages({
+ *   THIS_IS_A_MESSAGE: 'it has some content with {values}',
+ * });
+ *
+ * // SomeConsumer.tsx
+ * import someModuleMessages from 'SomeModule.messages.js';
+ * i18n.format(someModuleMessages.THIS_IS_A_MESSAGE, {values: "I'm a value!"});
+ * ```
+ *
+ * This transformer will only handle `SomeModule.messages.js`, and will output
+ * something like:
+ *
+ * ```typescript
+ * const {i18n} = require('@discord/intl');
+ * const _keys = ["a9fn23"];
+ * const _locales = {"en-US": () => require('./messages/en-US.messages.json')};
+ * const loader = createLoader(_keys, _locales);
+ * export default loader.getBinds();
+ * ```
+ *
+ * Notice how the message keys have been hashed into short keys, and the
+ * dynamic imports for each locale's data have been inserted automatically. In
+ * this example, the only locale is the source locale.
+ *
+ * The SWC and Babel transformers then take care of the second file,
+ * transforming the usage to use the hashed keys:
+ *
+ * ```typescript
+ * import someModuleMessages from 'SomeModule.messages.js';
+ * i18n.format(someModuleMessages["a9fn23"], {values: "i'm a value!"});
+ * ```
+ */
+class MessageDefinitionsTransformer {
+  /**
+   * @param {import('./types.d.ts').MessageDefinitionsTransformerOptions} options
+   */
+  constructor(options) {
+    this.options = options;
+  }
+
+  /**
+   * Returns a compiled string for an object that maps locale names to a
+   * dynamic require function for that locale, based on the supported locales
+   * that were determined for this file. The shape ends up as:
+   *
+   * ```typescript
+   * {
+   *   "en-US": () => import("path/to/en-US.json"),
+   * }
+   * ```
+   *
+   * @returns {string}
+   */
+  getLocaleRequireMap() {
+    const localeProperties = [];
+    for (const [locale, importPath] of Object.entries(this.options.localeMap)) {
+      // This assumes that the author has specified `importPath`
+      // as a properly-resolvable path for the bundler, which we can't easily
+      // enforce, unfortunately.
+      localeProperties.push(`"${locale}": () => ${this.options.getTranslationImport(importPath)}`);
+    }
+
+    return `{${localeProperties.join(',')}}`;
+  }
+
+  /**
+   * Returns the reduced, transformed output for this file. Currently not
+   * configurable, but could be told to include default messages or preserve
+   * information as necessary.
+   *
+   * @returns {string}
+   */
+  getOutput() {
+    return [
+      this.options.getPrelude?.() ?? '// No additional prelude was configured.',
+      `const {createLoader} = require('@discord/intl');`,
+      `const _keys = ${JSON.stringify(this.options.messageKeys)};`,
+      `const _locales = ${this.getLocaleRequireMap()};`,
+      'const loader = createLoader(_keys, _locales);',
+      'export default loader.getBinds();',
+    ].join('\n');
+  }
+}
+
+module.exports = {
+  MessageDefinitionsTransformer,
+};

package/src/types.d.ts

@@ -0,0 +1,87 @@
+import { IntlSourceFile } from '@discord/intl-message-database';
+
+export interface MessageDefinitionsTransformerOptions {
+  /**
+   * The list of hashed message keys that this file manages.
+   */
+  messageKeys: string[];
+  /**
+   * Map of locale names to import paths used for loading translations.
+   */
+  localeMap: Record<string, string>;
+  /**
+   * Function to create a prelude that gets injected at the start of the transformed file to set up
+   * anything needed for other injections later on.
+   */
+  getPrelude?: () => string;
+  /**
+   * Function to generate an import/require statement for the compiled asset file. All imports
+   * should be asynchronous (e.g., typically use `import` rather than `require`), but some platforms
+   * implement loading differently and may need different syntax. For example, React Native Assets
+   * are bundled using `require` statements to return an Asset ID, which can then be loaded
+   * asynchronously by some other code to get the actual content of the asset.
+   *
+   * The code created by this function must create a `Promise<{default: Record<string, any>}>`. In
+   * other words, a Promise for an object with a `default` key pointing to an object map of message
+   * keys to their values. For `import` statements, this is already the default. For `requires`, you
+   * may need to wrap the result with the `default` key, like `.then((data) => ({default: data}))`.
+   */
+  getTranslationImport(importPath: string): string;
+}
+
+/**
+ * The result of calling `processDefinitionsFile`, including the created source file, locale map,
+ * and more.
+ */
+export interface ProcessDefinitionsResult {
+  /**
+   * Direct source file from the database that was created or updated by this process.
+   */
+  sourceFile: IntlSourceFile;
+  /**
+   * The locale that was either determined from the sourceFile name or overridden by the options
+   * provided to this call.
+   */
+  locale: string;
+  /**
+   * The full list of message keys contained by the processed source file. While `sourceFile`
+   * contains a list of key _symbols_, this list contains all of the resolved strings for the
+   * hashed message keys.
+   */
+  hashedMessageKeys: string[];
+  /**
+   * Fully-resolved path to the translations directory that was scanned for entries for the
+   * source file.
+   */
+  translationsPath: string;
+  /**
+   * Map of locale names to file paths for all translations files that were discovered when scanning
+   * the configured `translationsPath`. Note that this _does not_ include the source locale, since
+   * it's target is often different between loaders (e.g., could be a virtual file, an asset that
+   * gets compiled separately, or use query parameters to control loader behavior when reusing the
+   * same file).
+   */
+  translationsLocaleMap: Record<string, string>;
+}
+
+/**
+ * The result of calling `processTranslationsFile`, including the created source file, locale map,
+ * and more.
+ */
+export interface ProcessTranslationsResult {
+  /**
+   * Direct source file from the database that was created or updated by this process.
+   */
+  sourceFile: IntlSourceFile;
+  /**
+   * The locale that was either determined from the sourceFile name or overridden by the options
+   * provided to this call.
+   */
+  locale: string;
+  /**
+   * The full list of message keys contained by the processed source file. While `sourceFile`
+   * contains a list of key _symbols_, this list contains all of the resolved strings for the
+   * hashed message keys.
+   */
+  hashedMessageKeys: string[];
+}

package/src/util.js

@@ -0,0 +1,55 @@
+const path = require('node:path');
+const fs = require('node:fs');
+const { isMessageTranslationsFile } = require('@discord/intl-message-database');
+
+const IGNORED_MESSAGE_FILE_PATTERNS = [/.*\.compiled.messages\..*/];
+
+/**
+ * Return the presumed locale for a translations file from it's name. The convention follows the
+ * format: `some/path/to/<locale>.messages.jsona`, so the locale is determined by taking the content
+ * of the basename up until the first `.`.
+ *
+ * @param {string} fileName
+ * @returns {string}
+ */
+function getLocaleFromTranslationsFileName(fileName) {
+  return path.basename(fileName).split('.')[0];
+}
+
+/**
+ * Scan the given `translationsPath` to discover all translation files that exist, returning them
+ * as a map from locale name to the path for importing.
+ *
+ * @param {string} translationsPath
+ * @returns {Record<string, string>}
+ */
+function findAllTranslationFiles(translationsPath) {
+  /** @type {Record<string, string>} */
+  const localeMap = {};
+
+  try {
+    const translationFiles = fs.readdirSync(translationsPath, { encoding: 'utf-8' });
+    for (const foundFile of translationFiles) {
+      const filePath = path.join(translationsPath, foundFile);
+      // Only include translation files, not definitions files.
+      if (!isMessageTranslationsFile(filePath)) continue;
+      // Some files are excluded, like pre-compiled artifacts.
+      if (IGNORED_MESSAGE_FILE_PATTERNS.some((pattern) => pattern.test(filePath))) continue;
+
+      const locale = getLocaleFromTranslationsFileName(filePath);
+      localeMap[locale] = filePath;
+    }
+  } catch (e) {
+    throw new Error(
+      `The translations directory ${translationsPath} was not found. No translations will be loaded for these messages`,
+    );
+  }
+
+  return localeMap;
+}
+
+module.exports = {
+  IGNORED_MESSAGE_FILE_PATTERNS,
+  findAllTranslationFiles,
+  getLocaleFromTranslationsFileName,
+};

package/src/watcher.js

@@ -0,0 +1,92 @@
+const path = require('node:path');
+
+const chokidar = require('chokidar');
+const fg = require('fast-glob');
+const debug = require('debug')('intl:metro-intl-transformer:watcher');
+const {
+  isMessageDefinitionsFile,
+  IntlCompiledMessageFormat,
+} = require('@discord/intl-message-database');
+
+const { database } = require('./database');
+const { processDefinitionsFile, precompileFileForLocale } = require('./processing');
+
+const ALWAYS_IGNORE_PATTERNS = [
+  // Ignore our own compiled message files, even though they shouldn't have a matching extension.
+  '*.compiled.messages.*',
+];
+// TODO: This should come from the database extension? Or Utilities? Unsure, but the extensions
+// should have some centralized location in general
+const MESSAGE_DEFINITION_FILE_PATTERNS = ['**/*.messages.js'];
+const DEFAULT_LOCALE = 'en-US';
+
+/**
+ * @param {string} filePath
+ * @param {string} assetExtension
+ */
+function processFile(filePath, assetExtension) {
+  debug(`Processing file: ${filePath}`);
+  if (!isMessageDefinitionsFile(filePath)) {
+    debug(`${filePath} is not a definitions file. Skipping processing`);
+    return;
+  }
+
+  try {
+    // Convert the file name from `.messages.js` to `.compiled.messages.jsona` for output.
+    const outputPath = filePath.replace(/\.messages\.js$/, `.compiled.messages.${assetExtension}`);
+    const result = processDefinitionsFile(filePath);
+    precompileFileForLocale(filePath, result.locale, { format: IntlCompiledMessageFormat.Json });
+
+    database.processDefinitionsFile(filePath);
+    database.precompile(filePath, DEFAULT_LOCALE, outputPath, IntlCompiledMessageFormat.Json);
+    debug(`Wrote definitions to: ${outputPath}`);
+  } catch (e) {
+    debug('[INTL Error] Failed to compile messages');
+    console.error(e);
+  }
+}
+
+/**
+ * @param {string[]} watchedFolders
+ * @param {{
+ *  watch?: boolean,
+ *  ignore?: string[],
+ *  assetExtension?: string
+ * }} options
+ */
+async function compileIntlMessageFiles(
+  watchedFolders,
+  { watch = true, ignore = [], assetExtension = 'json' } = {},
+) {
+  const ignoredPatterns = ignore.concat(ALWAYS_IGNORE_PATTERNS);
+  const globs = watchedFolders.flatMap((folder) =>
+    MESSAGE_DEFINITION_FILE_PATTERNS.map((pattern) => path.join(folder, pattern)),
+  );
+  debug(`Configured message file patterns:\n- ${globs.join('\n- ')}`);
+
+  // Perform one initial scan and compilation to ensure all files exist before Metro might try to
+  // resolve them.
+  debug('Scanning for initial messages files');
+  for await (const filePath of fg.stream(globs, {
+    ignore: ignoredPatterns,
+    absolute: true,
+    onlyFiles: true,
+  })) {
+    processFile(filePath.toString(), assetExtension);
+  }
+  debug('Initial message scan completed.');
+
+  if (watch) {
+    debug(`Setting up file watching for configured paths`);
+    chokidar
+      .watch(globs, { ignored: ignoredPatterns, ignoreInitial: true })
+      .on('all', (event, filePath) => {
+        debug(`Got event ${event} for ${filePath}`);
+        processFile(filePath, assetExtension);
+      });
+  } else {
+    debug('Not watching files because `watch` option was false');
+  }
+}
+
+module.exports = { compileIntlMessageFiles, ALWAYS_IGNORE_PATTERNS };