@discord/intl-loader-core 0.11.2-rc.3 → 0.11.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@discord/intl-loader-core",
-  "version": "0.11.2-rc.3",
+  "version": "0.11.3",
   "license": "MIT",
   "description": "Core utilities for writing loaders and transformers using @discord/intl",
   "author": "Jon Egeland",
@@ -28,8 +28,7 @@
   "dependencies": {
     "chokidar": "^3.6.0",
     "debug": "^4.3.6",
-    "fast-glob": "^3.3.2",
-    "@discord/intl-message-database": "0.11.2-rc.3"
+    "@discord/intl-message-database": "0.11.3"
   },
   "devDependencies": {
     "@types/debug": "^4.1.12",

package/src/processing.js

@@ -54,6 +54,18 @@ function buildTranslationsLocaleMap(sourcePath, sourceFile, translationsPath) {
 function findAllMessagesFiles(directories, defaultLocale = 'en-US') {
   return database.findAllMessagesFiles(directories, defaultLocale);
 }
+/**
+ * Given an arbitrary list of `files`, keep only those that can be treated as messages files, either
+ * definitions or translations. The returned list is a set of descriptors that can be processed by
+ * `processAllMessagesFiles`.
+ *
+ * @param {string[]} files
+ * @param {string} defaultLocale
+ * @returns {import('../types').IntlMessagesFileDescriptor[]}
+ */
+function filterAllMessagesFiles(files, defaultLocale = 'en-US') {
+  return database.filterAllMessagesFiles(files, defaultLocale);
+}
 
 /**
  * Iterate the given `files`, processing each one's content into the database. Processing is done
@@ -217,6 +229,7 @@ function generateTypeDefinitions(sourcePath, outputFile, allowNullability = fals
 
 module.exports = {
   findAllMessagesFiles,
+  filterAllMessagesFiles,
   processAllMessagesFiles,
   generateTypeDefinitions,
   precompileFileForLocale,

package/src/watcher.js

@@ -1,7 +1,6 @@
 const path = require('node:path');
 
 const chokidar = require('chokidar');
-const fg = require('fast-glob');
 const debug = require('debug')('intl:loader-core:watcher');
 const {
   isMessageDefinitionsFile,
@@ -9,93 +8,154 @@ const {
 } = require('@discord/intl-message-database');
 
 const { database } = require('./database');
-const { processDefinitionsFile, precompileFileForLocale } = require('./processing');
+const {
+  processDefinitionsFile,
+  precompileFileForLocale,
+  generateTypeDefinitions,
+  findAllMessagesFiles,
+  filterAllMessagesFiles,
+  processAllMessagesFiles,
+} = require('./processing');
 
 const ALWAYS_IGNORE_PATTERNS = [
   // Ignore our own compiled message files, even though they shouldn't have a matching extension.
   '*.compiled.messages.*',
+  '*.d.ts',
+  '*.d.ts.map',
 ];
 // TODO: This should come from the database extension? Or Utilities? Unsure, but the extensions
 // should have some centralized location in general
-const MESSAGE_DEFINITION_FILE_PATTERNS = ['**/*.messages.js'];
+const MESSAGES_FILE_PATTERNS = ['**/*.messages.*'];
 const DEFAULT_LOCALE = 'en-US';
 
 /**
- * @param {string} filePath
- * @param {string} assetExtension
- * @param {import('./processing').IntlPrecompileOptions} [options]
+ * Start a new filesystem watcher to discover and monitor messages files, allowing consumers to
+ * react to changes in a single, consistent way.
+ *
+ * @param {string[]} watchedFolders List of folders to watch recursively.
+ * @param {{
+ *  ignore?: string[],
+ *  skipInitial?: boolean,
+ *  persistent?: boolean,
+ *  once?: boolean,
+ * }} options
+ * @returns {Promise<{
+ *   on(event: string, callback: (event: string, path: string) => void): void
+ * }>}
  */
-function processFile(filePath, assetExtension, options = {}) {
-  const { format = IntlCompiledMessageFormat.KeylessJson, bundleSecrets = false } = options;
-  debug(`Processing file: ${filePath}`);
-  if (!isMessageDefinitionsFile(filePath)) {
-    debug(`${filePath} is not a definitions file. Skipping processing`);
-    return;
+async function watchMessagesFiles(
+  watchedFolders,
+  { ignore = [], skipInitial = false, once = true } = {},
+) {
+  debug('Pre-initializing database with all discoverable messages files');
+  const files = findAllMessagesFiles(watchedFolders);
+  processAllMessagesFiles(files);
+  if (once) {
+    debug('Performing one-shot watch because `once` was true');
+    // This is a weird hack around file watching just not working well on
+    // containers/CI. `add` events won't fire consistently from chokidar when
+    // it's not set to be persistent. So for a `once` run, we can just manually
+    // invoke the event for every file that was discovered on the initial scan.
+    return {
+      on(_, callback) {
+        for (const file of files) {
+          callback('add', file.filePath);
+        }
+      },
+    };
   }
 
-  try {
-    // Convert the file name from `.messages.js` to `.compiled.messages.jsona` for output.
-    const outputPath = filePath.replace(/\.messages\.js$/, `.compiled.messages.${assetExtension}`);
-    const result = processDefinitionsFile(filePath);
-    precompileFileForLocale(filePath, result.locale, undefined, {
-      format,
-      bundleSecrets,
-    });
+  const ignoredPatterns = ignore.concat(ALWAYS_IGNORE_PATTERNS);
+  const globs = watchedFolders.flatMap((folder) =>
+    MESSAGES_FILE_PATTERNS.map((pattern) => path.join(folder, pattern)),
+  );
 
-    database.processDefinitionsFile(filePath);
-    database.precompile(filePath, DEFAULT_LOCALE, outputPath, {
-      format: IntlCompiledMessageFormat.KeylessJson,
-      bundleSecrets,
+  debug(`Initializing watch for patterns:\n- ${globs.join('\n- ')}`);
+  const watcher = chokidar.watch(globs, {
+    ignored: ignoredPatterns,
+    ignoreInitial: skipInitial,
+    persistent: !once,
+  });
+  watcher.on('all', (event, path) => {
+    processAllMessagesFiles(filterAllMessagesFiles([path]));
+    debug(`watcher got event: ${event}, for path ${path}`);
+  });
+  for (const signal of ['SIGINT', 'SIGTERM', 'SIGQUIT']) {
+    process.on(signal, async () => {
+      debug(`Stopping watch due to`, signal);
+      await watcher.close();
+      debug('Watch stopped');
     });
-    debug(`Wrote definitions to: ${outputPath}`);
-  } catch (e) {
-    debug('[INTL Error] Failed to compile messages');
-    console.error(e);
   }
+
+  return watcher;
+}
+
+/**
+ * Watch the file system for any changes to message files and regenerate their types.
+ *
+ * @param {chokidar.FSWatcher} watcher
+ * @param {{
+ *  allowNullability?: boolean
+ * }} options
+ */
+async function generateMessageTypes(watcher, { allowNullability }) {
+  watcher.on('all', (_, filePath) => {
+    if (!isMessageDefinitionsFile(filePath)) return;
+    debug(`Regenerating types for ${filePath}`);
+    generateTypeDefinitions(filePath, undefined, allowNullability);
+  });
 }
 
 /**
- * @param {string[]} watchedFolders
+ * Watch the file system for changes to message _definitions_ files, and pre-process them into
+ * compiled assets (with the extension `.compiled.messages.${assetExtension}`).
+ *
+ * @param {chokidar.FSWatcher} watcher
  * @param {{
- *  watch?: boolean,
  *  ignore?: string[],
  *  assetExtension?: string
  *  precompileOptions?: import('./processing').IntlPrecompileOptions,
  * }} options
  */
-async function compileIntlMessageFiles(
-  watchedFolders,
-  { watch = true, ignore = [], assetExtension = 'json', precompileOptions = {} } = {},
+async function precompileMessageDefinitionsFiles(
+  watcher,
+  { assetExtension = 'json', precompileOptions = {} } = {},
 ) {
-  const ignoredPatterns = ignore.concat(ALWAYS_IGNORE_PATTERNS);
-  const globs = watchedFolders.flatMap((folder) =>
-    MESSAGE_DEFINITION_FILE_PATTERNS.map((pattern) => path.join(folder, pattern)),
-  );
-  debug(`Configured message file patterns:\n- ${globs.join('\n- ')}`);
+  watcher.on('all', (_, filePath) => {
+    const { format = IntlCompiledMessageFormat.KeylessJson, bundleSecrets = false } =
+      precompileOptions;
+    if (!isMessageDefinitionsFile(filePath)) return;
 
-  // Perform one initial scan and compilation to ensure all files exist before Metro might try to
-  // resolve them.
-  debug('Scanning for initial messages files');
-  for await (const filePath of fg.stream(globs, {
-    ignore: ignoredPatterns,
-    absolute: true,
-    onlyFiles: true,
-  })) {
-    processFile(filePath.toString(), assetExtension, precompileOptions);
-  }
-  debug('Initial message scan completed.');
+    debug(`Processing file: ${filePath}`);
+    try {
+      // Convert the file name from `.messages.js` to `.compiled.messages.jsona` for output.
+      const outputPath = filePath.replace(
+        /\.messages\.js$/,
+        `.compiled.messages.${assetExtension}`,
+      );
+      const result = processDefinitionsFile(filePath);
+      precompileFileForLocale(filePath, result.locale, undefined, {
+        format,
+        bundleSecrets,
+      });
 
-  if (watch) {
-    debug(`Setting up file watching for configured paths`);
-    chokidar
-      .watch(globs, { ignored: ignoredPatterns, ignoreInitial: true })
-      .on('all', (event, filePath) => {
-        debug(`Got event ${event} for ${filePath}`);
-        processFile(filePath, assetExtension, precompileOptions);
+      database.processDefinitionsFile(filePath);
+      database.precompile(filePath, DEFAULT_LOCALE, outputPath, {
+        format: IntlCompiledMessageFormat.KeylessJson,
+        bundleSecrets,
       });
-  } else {
-    debug('Not watching files because `watch` option was false');
-  }
+      debug(`Wrote definitions to: ${outputPath}`);
+    } catch (e) {
+      debug('[INTL Error] Failed to compile messages');
+      console.error(e);
+    }
+  });
 }
 
-module.exports = { compileIntlMessageFiles, ALWAYS_IGNORE_PATTERNS };
+module.exports = {
+  watchMessagesFiles,
+  precompileMessageDefinitionsFiles,
+  generateMessageTypes,
+  ALWAYS_IGNORE_PATTERNS,
+};

package/types/src/processing.d.ts

@@ -11,6 +11,16 @@ export type IntlPrecompileOptions = {
  * @returns {import('../types').IntlMessagesFileDescriptor[]}
  */
 export function findAllMessagesFiles(directories: string[], defaultLocale?: string): import("../types").IntlMessagesFileDescriptor[];
+/**
+ * Given an arbitrary list of `files`, keep only those that can be treated as messages files, either
+ * definitions or translations. The returned list is a set of descriptors that can be processed by
+ * `processAllMessagesFiles`.
+ *
+ * @param {string[]} files
+ * @param {string} defaultLocale
+ * @returns {import('../types').IntlMessagesFileDescriptor[]}
+ */
+export function filterAllMessagesFiles(files: string[], defaultLocale?: string): import("../types").IntlMessagesFileDescriptor[];
 /**
  * Iterate the given `files`, processing each one's content into the database. Processing is done
  * in parallel using native acceleration, and returns the list of all file names that were used.

package/types/src/watcher.d.ts

@@ -1,16 +1,52 @@
 /**
- * @param {string[]} watchedFolders
+ * Start a new filesystem watcher to discover and monitor messages files, allowing consumers to
+ * react to changes in a single, consistent way.
+ *
+ * @param {string[]} watchedFolders List of folders to watch recursively.
+ * @param {{
+ *  ignore?: string[],
+ *  skipInitial?: boolean,
+ *  persistent?: boolean,
+ *  once?: boolean,
+ * }} options
+ * @returns {Promise<{
+ *   on(event: string, callback: (event: string, path: string) => void): void
+ * }>}
+ */
+export function watchMessagesFiles(watchedFolders: string[], { ignore, skipInitial, once }?: {
+    ignore?: string[];
+    skipInitial?: boolean;
+    persistent?: boolean;
+    once?: boolean;
+}): Promise<{
+    on(event: string, callback: (event: string, path: string) => void): void;
+}>;
+/**
+ * Watch the file system for changes to message _definitions_ files, and pre-process them into
+ * compiled assets (with the extension `.compiled.messages.${assetExtension}`).
+ *
+ * @param {chokidar.FSWatcher} watcher
  * @param {{
- *  watch?: boolean,
  *  ignore?: string[],
  *  assetExtension?: string
  *  precompileOptions?: import('./processing').IntlPrecompileOptions,
  * }} options
  */
-export function compileIntlMessageFiles(watchedFolders: string[], { watch, ignore, assetExtension, precompileOptions }?: {
-    watch?: boolean;
+export function precompileMessageDefinitionsFiles(watcher: chokidar.FSWatcher, { assetExtension, precompileOptions }?: {
     ignore?: string[];
     assetExtension?: string;
     precompileOptions?: import("./processing").IntlPrecompileOptions;
 }): Promise<void>;
+/**
+ * Watch the file system for any changes to message files and regenerate their types.
+ *
+ * @param {chokidar.FSWatcher} watcher
+ * @param {{
+ *  allowNullability?: boolean
+ * }} options
+ */
+export function generateMessageTypes(watcher: chokidar.FSWatcher, { allowNullability }: {
+    allowNullability?: boolean;
+}): Promise<void>;
 export const ALWAYS_IGNORE_PATTERNS: string[];
+import chokidar = require("chokidar");