@discord/intl-loader-core 0.8.0 → 0.9.0-canary.66e0616

package/package.json

@@ -1,18 +1,22 @@
 {
   "name": "@discord/intl-loader-core",
-  "version": "0.8.0",
+  "version": "0.9.0-canary.66e0616",
   "license": "MIT",
   "description": "Core utilities for writing loaders and transformers using @discord/intl",
   "author": "Jon Egeland",
   "main": "index.js",
-  "types": "index.d.ts",
+  "types": "types/index.d.ts",
   "files": [
     "index.js",
-    "index.d.ts",
-    "src"
+    "types.d.ts",
+    "src",
+    "types"
   ],
   "exports": {
-    ".": "./index.js"
+    ".": {
+      "default": "./index.js",
+      "types": "./types/index.d.ts"
+    }
   },
   "repository": {
     "type": "git",
@@ -22,9 +26,14 @@
     "chokidar": "^3.6.0",
     "debug": "^4.3.6",
     "fast-glob": "^3.3.2",
-    "@discord/intl-message-database": "0.8.0"
+    "@discord/intl-message-database": "0.9.0-rc.2"
   },
   "devDependencies": {
-    "@types/debug": "^4.1.12"
+    "@types/debug": "^4.1.12",
+    "typescript": "*"
+  },
+  "scripts": {
+    "build": "tsc && cp types.d.ts types/",
+    "build:release": "tsc && cp types.d.ts types/"
   }
 }

package/src/database.js

@@ -1,4 +1,4 @@
-const {IntlMessagesDatabase} = require('@discord/intl-message-database');
+const { IntlMessagesDatabase } = require('@discord/intl-message-database');
 
 /**
  * A shared message database instance that's used and shared across all parts
@@ -6,4 +6,4 @@ const {IntlMessagesDatabase} = require('@discord/intl-message-database');
  */
 const database = new IntlMessagesDatabase();
 
-module.exports = {database};
+module.exports = { database };

package/src/processing.js

@@ -6,6 +6,13 @@ const { IntlCompiledMessageFormat } = require('@discord/intl-message-database');
 const { database } = require('./database');
 const { findAllTranslationFiles, getLocaleFromTranslationsFileName } = require('./util');
 
+/**
+ * @typedef {{
+ *   format?: IntlCompiledMessageFormat,
+ *   bundleSecrets?: boolean,
+ * }} IntlPrecompileOptions
+ */
+
 /**
  * @param {string} sourcePath
  * @param {import('@discord/intl-message-database').IntlSourceFile} sourceFile
@@ -43,7 +50,7 @@ function buildTranslationsLocaleMap(sourcePath, sourceFile, translationsPath) {
  *   processTranslations?: boolean,
  *   locale?: string
  * }=} options
- * @returns {import('./types').ProcessDefinitionsResult}
+ * @returns {import('../types').ProcessDefinitionsResult}
  */
 function processDefinitionsFile(sourcePath, sourceContent, options = {}) {
   const {
@@ -95,7 +102,7 @@ function processDefinitionsFile(sourcePath, sourceContent, options = {}) {
  * @param {{
  *   locale?: string,
  * }=} options
- * @returns {import('./types').ProcessTranslationsResult}
+ * @returns {import('../types').ProcessTranslationsResult}
  */
 function processTranslationsFile(sourcePath, sourceContent, options = {}) {
   const { locale = getLocaleFromTranslationsFileName(sourcePath) } = options;
@@ -129,20 +136,21 @@ function processTranslationsFile(sourcePath, sourceContent, options = {}) {
  * but if `outputFile` is given then the content will be written directly to the file and the
  * function becomes `void`.
  *
+ * Compiling automatically handles filtering out messages based on the meta information like
+ * `translate`, `secret`, and `bundleSecrets`, to ensure that all consumers apply these values
+ * accurately and consistently.
+ *
  * @param {string} sourcePath
  * @param {string} locale
- * @param {{
- *   format?: IntlCompiledMessageFormat,
- *   outputFile?: string
- * }=} options
+ * @param {string=} outputFile
+ * @param {IntlPrecompileOptions} [options]
  *
  * @returns {Buffer | void}
  */
-function precompileFileForLocale(sourcePath, locale, options = {}) {
-  const { format = IntlCompiledMessageFormat.KeylessJson, outputFile } = options;
+function precompileFileForLocale(sourcePath, locale, outputFile, options = {}) {
   return outputFile != null
-    ? database.precompile(sourcePath, locale, outputFile, format)
-    : database.precompileToBuffer(sourcePath, locale, format);
+    ? database.precompile(sourcePath, locale, outputFile, options)
+    : database.precompileToBuffer(sourcePath, locale, options);
 }
 
 /**

package/src/transformer.js

@@ -64,7 +64,7 @@
  */
 class MessageDefinitionsTransformer {
   /**
-   * @param {import('./types.d.ts').MessageDefinitionsTransformerOptions} options
+   * @param {import('../types.d.ts').MessageDefinitionsTransformerOptions} options
    */
   constructor(options) {
     this.options = options;

package/src/watcher.js

@@ -2,7 +2,7 @@ const path = require('node:path');
 
 const chokidar = require('chokidar');
 const fg = require('fast-glob');
-const debug = require('debug')('intl:metro-intl-transformer:watcher');
+const debug = require('debug')('intl:loader-core:watcher');
 const {
   isMessageDefinitionsFile,
   IntlCompiledMessageFormat,
@@ -23,8 +23,10 @@ const DEFAULT_LOCALE = 'en-US';
 /**
  * @param {string} filePath
  * @param {string} assetExtension
+ * @param {import('./processing').IntlPrecompileOptions} [options]
  */
-function processFile(filePath, assetExtension) {
+function processFile(filePath, assetExtension, options = {}) {
+  const { format = IntlCompiledMessageFormat.KeylessJson, bundleSecrets = false } = options;
   debug(`Processing file: ${filePath}`);
   if (!isMessageDefinitionsFile(filePath)) {
     debug(`${filePath} is not a definitions file. Skipping processing`);
@@ -35,17 +37,16 @@ function processFile(filePath, assetExtension) {
     // Convert the file name from `.messages.js` to `.compiled.messages.jsona` for output.
     const outputPath = filePath.replace(/\.messages\.js$/, `.compiled.messages.${assetExtension}`);
     const result = processDefinitionsFile(filePath);
-    precompileFileForLocale(filePath, result.locale, {
-      format: IntlCompiledMessageFormat.KeylessJson,
+    precompileFileForLocale(filePath, result.locale, undefined, {
+      format,
+      bundleSecrets,
     });
 
     database.processDefinitionsFile(filePath);
-    database.precompile(
-      filePath,
-      DEFAULT_LOCALE,
-      outputPath,
-      IntlCompiledMessageFormat.KeylessJson,
-    );
+    database.precompile(filePath, DEFAULT_LOCALE, outputPath, {
+      format: IntlCompiledMessageFormat.KeylessJson,
+      bundleSecrets,
+    });
     debug(`Wrote definitions to: ${outputPath}`);
   } catch (e) {
     debug('[INTL Error] Failed to compile messages');
@@ -59,11 +60,12 @@ function processFile(filePath, assetExtension) {
  *  watch?: boolean,
  *  ignore?: string[],
  *  assetExtension?: string
+ *  precompileOptions?: import('./processing').IntlPrecompileOptions,
  * }} options
  */
 async function compileIntlMessageFiles(
   watchedFolders,
-  { watch = true, ignore = [], assetExtension = 'json' } = {},
+  { watch = true, ignore = [], assetExtension = 'json', precompileOptions = {} } = {},
 ) {
   const ignoredPatterns = ignore.concat(ALWAYS_IGNORE_PATTERNS);
   const globs = watchedFolders.flatMap((folder) =>
@@ -79,7 +81,7 @@ async function compileIntlMessageFiles(
     absolute: true,
     onlyFiles: true,
   })) {
-    processFile(filePath.toString(), assetExtension);
+    processFile(filePath.toString(), assetExtension, precompileOptions);
   }
   debug('Initial message scan completed.');
 
@@ -89,7 +91,7 @@ async function compileIntlMessageFiles(
       .watch(globs, { ignored: ignoredPatterns, ignoreInitial: true })
       .on('all', (event, filePath) => {
         debug(`Got event ${event} for ${filePath}`);
-        processFile(filePath, assetExtension);
+        processFile(filePath, assetExtension, precompileOptions);
       });
   } else {
     debug('Not watching files because `watch` option was false');

package/types/index.d.ts

@@ -0,0 +1,14 @@
+import { IntlCompiledMessageFormat } from "@discord/intl-message-database";
+import { MessageDefinitionsTransformer } from "./src/transformer";
+import { database } from "./src/database";
+import { findAllTranslationFiles } from "./src/util";
+import { getLocaleFromTranslationsFileName } from "./src/util";
+import { generateTypeDefinitions } from "./src/processing";
+import { hashMessageKey } from "@discord/intl-message-database";
+import { isMessageDefinitionsFile } from "@discord/intl-message-database";
+import { isMessageTranslationsFile } from "@discord/intl-message-database";
+import { processDefinitionsFile } from "./src/processing";
+import { processTranslationsFile } from "./src/processing";
+import { precompileFileForLocale } from "./src/processing";
+import watcher = require("./src/watcher");
+export { IntlCompiledMessageFormat, MessageDefinitionsTransformer, database, findAllTranslationFiles, getLocaleFromTranslationsFileName, generateTypeDefinitions, hashMessageKey, isMessageDefinitionsFile, isMessageTranslationsFile, processDefinitionsFile, processTranslationsFile, precompileFileForLocale, watcher };

package/types/src/database.d.ts

@@ -0,0 +1,6 @@
+/**
+ * A shared message database instance that's used and shared across all parts
+ * of the loader and plugin together.
+ */
+export const database: IntlMessagesDatabase;
+import { IntlMessagesDatabase } from "@discord/intl-message-database";

package/types/src/processing.d.ts

@@ -0,0 +1,74 @@
+export type IntlPrecompileOptions = {
+    format?: IntlCompiledMessageFormat;
+    bundleSecrets?: boolean;
+};
+/**
+ * Generate a `.d.ts` file containing TypeScript type definitions for all of the messages defined in
+ * `sourcePath`. This method does not process `sourcePath` at all, meaning it expects the database
+ * to already know about the source, as well as all of the related translations to create an
+ * accurate typescript definition for each message.
+ *
+ * If not given, `outputFile` will default to the same path as `sourcePath`, with the last extension
+ * replaced by `.d.ts`. For example, a file like `SomeMessages.Other.messages.js` would become
+ * `SomeMessages.Other.messages.d.ts`.
+ *
+ * If `allowNullability` is set, the generated types for variables within messages will allow
+ * `null` and `undefined` for most value types, as well as looser restrictions on typing, such as
+ * allowing `string | number` for number variables.
+ *
+ * Returns `true` if the types were successfully generated, or `false` otherwise, such as if the
+ * source file is not already in the database.
+ *
+ * @param {string} sourcePath
+ * @param {string=} outputFile
+ * @param {boolean=} allowNullability
+ * @returns {boolean}
+ */
+export function generateTypeDefinitions(sourcePath: string, outputFile?: string | undefined, allowNullability?: boolean | undefined): boolean;
+/**
+ * Precompile the messages defined in the given `sourcePath` using the value of the translation for
+ * that message in the given `locale`. `format` specifies which serialization format the result
+ * will be written in.
+ *
+ * By default, the compiled content will be returned as a Buffer containing the serialized string,
+ * but if `outputFile` is given then the content will be written directly to the file and the
+ * function becomes `void`.
+ *
+ * Compiling automatically handles filtering out messages based on the meta information like
+ * `translate`, `secret`, and `bundleSecrets`, to ensure that all consumers apply these values
+ * accurately and consistently.
+ *
+ * @param {string} sourcePath
+ * @param {string} locale
+ * @param {string=} outputFile
+ * @param {IntlPrecompileOptions} [options]
+ *
+ * @returns {Buffer | void}
+ */
+export function precompileFileForLocale(sourcePath: string, locale: string, outputFile?: string | undefined, options?: IntlPrecompileOptions | undefined): Buffer | void;
+/**
+ * @param {string} sourcePath
+ * @param {string=} sourceContent
+ * @param {{
+ *   processTranslations?: boolean,
+ *   locale?: string
+ * }=} options
+ * @returns {import('../types').ProcessDefinitionsResult}
+ */
+export function processDefinitionsFile(sourcePath: string, sourceContent?: string | undefined, options?: {
+    processTranslations?: boolean;
+    locale?: string;
+} | undefined): import("../types").ProcessDefinitionsResult;
+/**
+ *
+ * @param {string} sourcePath
+ * @param {string=} sourceContent
+ * @param {{
+ *   locale?: string,
+ * }=} options
+ * @returns {import('../types').ProcessTranslationsResult}
+ */
+export function processTranslationsFile(sourcePath: string, sourceContent?: string | undefined, options?: {
+    locale?: string;
+} | undefined): import("../types").ProcessTranslationsResult;
+import { IntlCompiledMessageFormat } from "@discord/intl-message-database";

package/types/src/transformer.d.ts

@@ -0,0 +1,106 @@
+/**
+ * Common class for parsing and transforming the content of a messages
+ * definition file (e.g., "SomeFeature.messages.js") into a production-ready
+ * version, with message keys obfuscated, loading harnesses configured, and
+ * more.
+ *
+ * This transformation is intended to be used alongside the _consumer_
+ * transforms implemented as SWC and Babel plugins, which transform the
+ * callsites for messages into matching formats. Consider this input example:
+ *
+ * ```typescript
+ * // SomeModule.messages.js
+ * import {defineMessages} from '@discord/intl';
+ *
+ * export default defineMessages({
+ *   THIS_IS_A_MESSAGE: 'it has some content with {values}',
+ * });
+ *
+ * // SomeConsumer.tsx
+ * import someModuleMessages from 'SomeModule.messages.js';
+ * i18n.format(someModuleMessages.THIS_IS_A_MESSAGE, {values: "I'm a value!"});
+ * ```
+ *
+ * This transformer will only handle `SomeModule.messages.js`, and will output
+ * something like:
+ *
+ * ```typescript
+ * const {i18n} = require('@discord/intl');
+ * const _keys = ["a9fn23"];
+ * const _locales = {"en-US": () => require('./messages/en-US.messages.json')};
+ * export const messagesLoader = createLoader(_keys, _locales);
+ * export default messagesLoader.getBinds();
+ * ```
+ *
+ * Notice how the message keys have been hashed into short keys, and the
+ * dynamic imports for each locale's data have been inserted automatically. In
+ * this example, the only locale is the source locale.
+ *
+ * The SWC and Babel transformers then take care of the second file,
+ * transforming the usage to use the hashed keys:
+ *
+ * ```typescript
+ * import someModuleMessages from 'SomeModule.messages.js';
+ * i18n.format(someModuleMessages["a9fn23"], {values: "i'm a value!"});
+ * ```
+ *
+ * The transformed file also contains a named export for `messagesLoader`,
+ * which consumers can use to query and update the loading state for the
+ * messages managed by that loader, including waiting for a locale to be
+ * loaded, kicking off new loads, and more:
+ *
+ * ```typescript
+ * import {messagesLoader} from 'SomeModule.messages.js';
+ * // Wait for the loader to be initialized with default messages
+ * await messagesLoader.waitForDefaultLocaleLoaded();
+ * // Wait for a specific locale to load, starting the load if it
+ * // is not yet in progress.
+ * await messagesLoader.waitForLocaleLoaded('fr');
+ * // In hot-reloading environments, use the second `requireCurrent`
+ * // parameter to wait for the latest data, even if a value already
+ * // exists.
+ * const loaded = messagesLoader.isLocaleLoaded('fr', true);
+ * ```
+ */
+export class MessageDefinitionsTransformer {
+    /**
+     * @param {import('../types.d.ts').MessageDefinitionsTransformerOptions} options
+     */
+    constructor(options: import("../types.d.ts").MessageDefinitionsTransformerOptions);
+    options: import("../types.d.ts").MessageDefinitionsTransformerOptions;
+    loaderName: string;
+    /**
+     * Returns a compiled string for an object that maps locale names to a
+     * dynamic require function for that locale, based on the supported locales
+     * that were determined for this file. The shape ends up as:
+     *
+     * ```typescript
+     * {
+     *   "en-US": () => import("path/to/en-US.json"),
+     * }
+     * ```
+     *
+     * @returns {string}
+     */
+    getLocaleRequireMap(): string;
+    /**
+     * Return a map of key hashes to their original values, as well as a plain-text map of locales
+     * to the file names that they import from.
+     *
+     * @returns {string[]}
+     */
+    debugModeSetup(): string[];
+    /**
+     * Return the lines to export fields from this module, as determined by the `exportMode` on this
+     * transformer.
+     */
+    exportFields(): string[];
+    /**
+     * Returns the reduced, transformed output for this file. Currently not
+     * configurable, but could be told to include default messages or preserve
+     * information as necessary.
+     *
+     * @returns {string}
+     */
+    getOutput(): string;
+}

package/types/src/util.d.ts

@@ -0,0 +1,18 @@
+export const IGNORED_MESSAGE_FILE_PATTERNS: RegExp[];
+/**
+ * Scan the given `translationsPath` to discover all translation files that exist, returning them
+ * as a map from locale name to the path for importing.
+ *
+ * @param {string} translationsPath
+ * @returns {Record<string, string> | Error}
+ */
+export function findAllTranslationFiles(translationsPath: string): Record<string, string> | Error;
+/**
+ * Return the presumed locale for a translations file from it's name. The convention follows the
+ * format: `some/path/to/<locale>.messages.jsona`, so the locale is determined by taking the content
+ * of the basename up until the first `.`.
+ *
+ * @param {string} fileName
+ * @returns {string}
+ */
+export function getLocaleFromTranslationsFileName(fileName: string): string;

package/types/src/watcher.d.ts

@@ -0,0 +1,16 @@
+/**
+ * @param {string[]} watchedFolders
+ * @param {{
+ *  watch?: boolean,
+ *  ignore?: string[],
+ *  assetExtension?: string
+ *  precompileOptions?: import('./processing').IntlPrecompileOptions,
+ * }} options
+ */
+export function compileIntlMessageFiles(watchedFolders: string[], { watch, ignore, assetExtension, precompileOptions }?: {
+    watch?: boolean;
+    ignore?: string[];
+    assetExtension?: string;
+    precompileOptions?: import("./processing").IntlPrecompileOptions;
+}): Promise<void>;
+export const ALWAYS_IGNORE_PATTERNS: string[];

package/types/types.d.ts

@@ -0,0 +1,111 @@
+import { IntlSourceFile } from '@discord/intl-message-database';
+
+export interface MessageDefinitionsTransformerOptions {
+  /**
+   * The map of message keys that this file manages to their original values. By default, only the
+   * keys of this map are used (the hashed names), but in debug mode the values will also be
+   * included in the transformed file to provide context in errors and warnings.
+   */
+  messageKeys: Record<string, string>;
+  /**
+   * Map of locale names to import paths used for loading translations.
+   */
+  localeMap: Record<string, string>;
+  /**
+   * Default locale to use for the runtime loader. This is almost always the locale of the source
+   * file being transformed, but can be set explicitly to something else for special cases.
+   */
+  defaultLocale: string;
+  /**
+   * Function to create a prelude that gets injected at the start of the transformed file to set up
+   * anything needed for other injections later on.
+   */
+  getPrelude?: () => string;
+  /**
+   * Function to generate an import/require statement for the compiled asset file. All imports
+   * should be asynchronous (e.g., typically use `import` rather than `require`), but some platforms
+   * implement loading differently and may need different syntax. For example, React Native Assets
+   * are bundled using `require` statements to return an Asset ID, which can then be loaded
+   * asynchronously by some other code to get the actual content of the asset.
+   *
+   * The code created by this function must create a `Promise<{default: Record<string, any>}>`. In
+   * other words, a Promise for an object with a `default` key pointing to an object map of message
+   * keys to their values. For `import` statements, this is already the default. For `requires`, you
+   * may need to wrap the result with the `default` key, like `.then((data) => ({default: data}))`.
+   */
+  getTranslationImport(importPath: string): string;
+
+  /**
+   * Whether to include additional information about keys and source files in the transformed loader
+   * code to provide context for debugging in errors and warning messages.
+   */
+  debug?: boolean;
+  /**
+   * Control how value exports are written in the transformed file to match what any downstream
+   * transformer or interpreter may expect. `esm` will leave exports as `export default` and other
+   * module features supported in ES6 and onward. `commonjs` will explicitly use `module.exports`
+   * with a single object containing `default` as the key for a default export. `transpiledEsModule`
+   * does the same, but includes the compatibility field `__esModule` to indicate that the module
+   * "was transpiled" to this syntax.
+   *
+   * @default esm
+   */
+  exportMode?: 'esm' | 'commonjs' | 'transpiledEsModule';
+}
+
+/**
+ * The result of calling `processDefinitionsFile`, including the created source file, locale map,
+ * and more.
+ */
+export interface ProcessDefinitionsResult {
+  /**
+   * Direct source file from the database that was created or updated by this process.
+   */
+  sourceFile: IntlSourceFile;
+  /**
+   * The locale that was either determined from the sourceFile name or overridden by the options
+   * provided to this call.
+   */
+  locale: string;
+  /**
+   * The full map of message keys contained by the processed source file to their original values.
+   * While `sourceFile` contains a list of key _symbols_, this list contains all of the
+   * resolved strings for the hashed message keys.
+   */
+  messageKeys: Record<string, string>;
+  /**
+   * Fully-resolved path to the translations directory that was scanned for entries for the
+   * source file.
+   */
+  translationsPath: string;
+  /**
+   * Map of locale names to file paths for all translations files that were discovered when scanning
+   * the configured `translationsPath`. Note that this _does not_ include the source locale, since
+   * it's target is often different between loaders (e.g., could be a virtual file, an asset that
+   * gets compiled separately, or use query parameters to control loader behavior when reusing the
+   * same file).
+   */
+  translationsLocaleMap: Record<string, string>;
+}
+
+/**
+ * The result of calling `processTranslationsFile`, including the created source file, locale map,
+ * and more.
+ */
+export interface ProcessTranslationsResult {
+  /**
+   * Direct source file from the database that was created or updated by this process.
+   */
+  sourceFile: IntlSourceFile;
+  /**
+   * The full map of message keys contained by the processed source file to their original values.
+   * While `sourceFile` contains a list of key _symbols_, this list contains all of the
+   * resolved strings for the hashed message keys.
+   */
+  messageKeys: Record<string, string>;
+  /**
+   * The locale that was either determined from the sourceFile name or overridden by the options
+   * provided to this call.
+   */
+  locale: string;
+}

package/index.d.ts

@@ -1,17 +0,0 @@
-export {
-  hashMessageKey,
-  isMessageDefinitionsFile,
-  isMessageTranslationsFile,
-  IntlCompiledMessageFormat,
-} from '@discord/intl-message-database';
-
-export { database } from './src/database';
-export {
-  generateTypeDefinitions,
-  processDefinitionsFile,
-  processTranslationsFile,
-  precompileFileForLocale,
-} from './src/processing';
-export { MessageDefinitionsTransformer } from './src/transformer';
-export { findAllTranslationFiles, getLocaleFromTranslationsFileName } from './src/util';
-export * as watcher from './src/watcher';