@discord/intl-loader-core 0.21.1 → 0.22.0-rc.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@discord/intl-loader-core",
-  "version": "0.21.1",
+  "version": "0.22.0-rc.0",
   "license": "MIT",
   "description": "Core utilities for writing loaders and transformers using @discord/intl",
   "author": "Jon Egeland",
@@ -28,7 +28,7 @@
   "dependencies": {
     "chokidar": "^3.6.0",
     "debug": "^4.3.6",
-    "@discord/intl-message-database": "0.21.1"
+    "@discord/intl-message-database": "0.22.0-rc.0"
   },
   "devDependencies": {
     "@types/debug": "^4.1.12",

package/src/processing.js

@@ -7,6 +7,13 @@ const { database } = require('./database');
 const { findAllTranslationFiles, getLocaleFromTranslationsFileName } = require('./util');
 
 /**
+ * @import {
+ *   IntlMessagesFileDescriptor,
+ *   IntlSourceFileInsertionData,
+ *   ProcessDefinitionsResult,
+ *   ProcessTranslationsResult
+ * } from '../types';
+ *
  * @typedef {{
  *   format?: IntlCompiledMessageFormat,
  *   bundleSecrets?: boolean,
@@ -63,7 +70,6 @@ function findAllDefinitionsFilesForTranslations(filePath) {
   for (const file of sourceFiles) {
     const source = database.getSourceFile(file);
     if (source.type === 'definition' && source.meta.translationsPath === expectedTranslationsPath) {
-      console.log(file, source.meta);
       relevantPaths.push(source.file);
     }
   }
@@ -89,7 +95,7 @@ function findAllMessagesFiles(directories, defaultLocale = 'en-US') {
  *
  * @param {string[]} files
  * @param {string} defaultLocale
- * @returns {import('../types').IntlMessagesFileDescriptor[]}
+ * @returns {IntlMessagesFileDescriptor[]}
  */
 function filterAllMessagesFiles(files, defaultLocale = 'en-US') {
   return database.filterAllMessagesFiles(files, defaultLocale);
@@ -97,14 +103,15 @@ function filterAllMessagesFiles(files, defaultLocale = 'en-US') {
 
 /**
  * Iterate the given `files`, processing each one's content into the database. Processing is done
- * in parallel using native acceleration, and returns the list of all file names that were used.
+ * in parallel using native acceleration, and returns an object describing all of the changes that
+ * were processed.
  *
  * Note that this method _only_ operates by reading file contents from the system, it is not
  * possible to supply preloaded content through a Buffer in the same way as `processDefinitionsFile`
  * with a `sourceContent` argument.
  *
- * @param {import('../types').IntlMessagesFileDescriptor[]} files
- * @returns {import('../types').IntlMultiProcessingResult}
+ * @param {IntlMessagesFileDescriptor[]} files
+ * @returns {IntlSourceFileInsertionData[]}
  */
 function processAllMessagesFiles(files) {
   return database.processAllMessagesFiles(files);
@@ -117,7 +124,7 @@ function processAllMessagesFiles(files) {
  *   processTranslations?: boolean,
  *   locale?: string
  * }=} options
- * @returns {import('../types').ProcessDefinitionsResult}
+ * @returns {ProcessDefinitionsResult}
  */
 function processDefinitionsFile(sourcePath, sourceContent, options = {}) {
   const {
@@ -127,10 +134,15 @@ function processDefinitionsFile(sourcePath, sourceContent, options = {}) {
   } = options;
   debug(`[${sourcePath}] Processing definitions with locale "${locale}"`);
 
+  /** @type {IntlSourceFileInsertionData} */
+  let result;
   if (sourceContent != null) {
-    database.processDefinitionsFileContent(sourcePath, sourceContent, locale);
+    result = database.processDefinitionsFileContent(sourcePath, sourceContent, locale);
   } else {
-    database.processDefinitionsFile(sourcePath, locale);
+    result = database.processDefinitionsFile(sourcePath, locale);
+  }
+  if (result.errors.length > 0) {
+    return { succeeded: false, errors: result.errors };
   }
 
   const sourceFile = database.getSourceFile(sourcePath);
@@ -151,10 +163,15 @@ function processDefinitionsFile(sourcePath, sourceContent, options = {}) {
 
   if (processTranslations) {
     const translationsResult = database.processAllTranslationFiles(translationsLocaleMap);
+    const allErrors = translationsResult.flatMap((result) => result.errors);
+    if (allErrors.length > 0) {
+      return { succeeded: false, errors: allErrors };
+    }
     debug('[${sourcePath}] Finished processing all translations: %O', translationsResult);
   }
 
   return {
+    succeeded: true,
     sourceFile,
     locale,
     messageKeys,
@@ -170,14 +187,20 @@ function processDefinitionsFile(sourcePath, sourceContent, options = {}) {
  * @param {{
  *   locale?: string,
  * }=} options
- * @returns {import('../types').ProcessTranslationsResult}
+ * @returns {ProcessTranslationsResult}
  */
 function processTranslationsFile(sourcePath, sourceContent, options = {}) {
   const { locale = getLocaleFromTranslationsFileName(sourcePath) } = options;
+
+  /** @type {IntlSourceFileInsertionData} */
+  let result;
   if (sourceContent) {
-    database.processTranslationFileContent(sourcePath, locale, sourceContent);
+    result = database.processTranslationFileContent(sourcePath, locale, sourceContent);
   } else {
-    database.processTranslationFile(sourcePath, locale);
+    result = database.processTranslationFile(sourcePath, locale);
+  }
+  if (result.errors.length > 0) {
+    return { succeeded: false, errors: result.errors };
   }
 
   const sourceFile = database.getSourceFile(sourcePath);
@@ -189,6 +212,7 @@ function processTranslationsFile(sourcePath, sourceContent, options = {}) {
   }
 
   return {
+    succeeded: true,
     sourceFile,
     locale,
     messageKeys: database.getSourceFileKeyMap(sourcePath),

package/src/watcher.js

@@ -30,7 +30,7 @@ const DEFAULT_LOCALE = 'en-US';
 
 /**
  * @typedef {{
- *   on(event: string, callback: (event: string, path: string) => void): void
+ *   on(event: string, callback: (event: string, path: string) => void): void,
  * }} BasicWatcher
  */
 
@@ -49,11 +49,15 @@ const DEFAULT_LOCALE = 'en-US';
  */
 async function watchMessagesFiles(
   watchedFolders,
-  { ignore = [], skipInitial = false, once = true } = {},
+  { ignore = [], skipInitial = false, once = true } = {}
 ) {
   debug('Pre-initializing database with all discoverable messages files');
   const files = findAllMessagesFiles(watchedFolders);
-  processAllMessagesFiles(files);
+  const result = processAllMessagesFiles(files);
+  const reprocessQueue = result
+    .filter((data) => data.errors.length > 0)
+    .map((data) => data.fileKey);
+
   if (once) {
     debug('Performing one-shot watch because `once` was true');
     // This is a weird hack around file watching just not working well on
@@ -71,7 +75,7 @@ async function watchMessagesFiles(
 
   const ignoredPatterns = ignore.concat(ALWAYS_IGNORE_PATTERNS);
   const globs = watchedFolders.flatMap((folder) =>
-    MESSAGES_FILE_PATTERNS.map((pattern) => path.join(folder, pattern)),
+    MESSAGES_FILE_PATTERNS.map((pattern) => path.join(folder, pattern))
   );
 
   debug(`Initializing watch for patterns:\n- ${globs.join('\n- ')}`);
@@ -80,8 +84,27 @@ async function watchMessagesFiles(
     ignoreInitial: skipInitial,
     persistent: !once,
   });
+
   watcher.on('all', (event, path) => {
-    processAllMessagesFiles(filterAllMessagesFiles([path]));
+    const files = filterAllMessagesFiles([path]);
+    if (files.length === 0) return;
+
+    // Process the file that changed first, then reprocess all of the queued files from previous
+    // failures to hopefully get them to process successfully again.
+    const [result] = processAllMessagesFiles(files);
+    const failed = result.errors.length > 0 ? [result.fileKey] : [];
+    const secondResult = processAllMessagesFiles(
+      filterAllMessagesFiles([...reprocessQueue, ...failed])
+    );
+    // For any files that are still failing, keep them in the queue for next time.
+    const secondFailures = secondResult
+      .filter((data) => data.errors.length > 0)
+      .map((data) => data.fileKey);
+    if (secondFailures.length > 0) {
+      debug('Failed to re-process files: %O. Next update will try again.', secondFailures);
+    }
+    reprocessQueue.splice(0, Infinity, ...secondFailures);
+
     debug(`watcher got event: ${event}, for path ${path}`);
   });
   for (const signal of ['SIGINT', 'SIGTERM', 'SIGQUIT']) {
@@ -121,7 +144,7 @@ async function generateMessageTypes(watcher) {
  */
 async function precompileMessageDefinitionsFiles(
   watcher,
-  { assetExtension = 'json', precompileOptions = {} } = {},
+  { assetExtension = 'json', precompileOptions = {} } = {}
 ) {
   watcher.on('all', (_, filePath) => {
     const { format = IntlCompiledMessageFormat.KeylessJson, bundleSecrets = false } =
@@ -133,19 +156,20 @@ async function precompileMessageDefinitionsFiles(
       // Convert the file name from `.messages.js` to `.compiled.messages.jsona` for output.
       const outputPath = filePath.replace(
         /\.messages\.js$/,
-        `.compiled.messages.${assetExtension}`,
+        `.compiled.messages.${assetExtension}`
       );
       const result = processDefinitionsFile(filePath);
+      if (!result.succeeded) {
+        debug('[INTL Error] Failed to compile messages');
+        console.error(result.errors);
+        return;
+      }
+
       precompileFileForLocale(filePath, result.locale, undefined, {
         format,
         bundleSecrets,
       });
 
-      database.processDefinitionsFile(filePath);
-      database.precompile(filePath, DEFAULT_LOCALE, outputPath, {
-        format: IntlCompiledMessageFormat.KeylessJson,
-        bundleSecrets,
-      });
       debug(`Wrote definitions to: ${outputPath}`);
     } catch (e) {
       debug('[INTL Error] Failed to compile messages');

package/types/src/processing.d.ts

@@ -32,21 +32,22 @@ export function findAllMessagesFiles(directories: string[], defaultLocale?: stri
  *
  * @param {string[]} files
  * @param {string} defaultLocale
- * @returns {import('../types').IntlMessagesFileDescriptor[]}
+ * @returns {IntlMessagesFileDescriptor[]}
  */
-export function filterAllMessagesFiles(files: string[], defaultLocale?: string): import("../types").IntlMessagesFileDescriptor[];
+export function filterAllMessagesFiles(files: string[], defaultLocale?: string): IntlMessagesFileDescriptor[];
 /**
  * Iterate the given `files`, processing each one's content into the database. Processing is done
- * in parallel using native acceleration, and returns the list of all file names that were used.
+ * in parallel using native acceleration, and returns an object describing all of the changes that
+ * were processed.
  *
  * Note that this method _only_ operates by reading file contents from the system, it is not
  * possible to supply preloaded content through a Buffer in the same way as `processDefinitionsFile`
  * with a `sourceContent` argument.
  *
- * @param {import('../types').IntlMessagesFileDescriptor[]} files
- * @returns {import('../types').IntlMultiProcessingResult}
+ * @param {IntlMessagesFileDescriptor[]} files
+ * @returns {IntlSourceFileInsertionData[]}
  */
-export function processAllMessagesFiles(files: import("../types").IntlMessagesFileDescriptor[]): import("../types").IntlMultiProcessingResult;
+export function processAllMessagesFiles(files: IntlMessagesFileDescriptor[]): IntlSourceFileInsertionData[];
 /**
  * Generate a `.d.ts` file containing TypeScript type definitions for all of the messages defined in
  * `sourcePath`. This method does not process `sourcePath` at all, meaning it expects the database
@@ -93,12 +94,12 @@ export function precompileFileForLocale(sourcePath: string, locale: string, outp
  *   processTranslations?: boolean,
  *   locale?: string
  * }=} options
- * @returns {import('../types').ProcessDefinitionsResult}
+ * @returns {ProcessDefinitionsResult}
  */
 export function processDefinitionsFile(sourcePath: string, sourceContent?: string | undefined, options?: {
     processTranslations?: boolean;
     locale?: string;
-} | undefined): import("../types").ProcessDefinitionsResult;
+} | undefined): ProcessDefinitionsResult;
 /**
  *
  * @param {string} sourcePath
@@ -106,9 +107,13 @@ export function processDefinitionsFile(sourcePath: string, sourceContent?: strin
  * @param {{
  *   locale?: string,
  * }=} options
- * @returns {import('../types').ProcessTranslationsResult}
+ * @returns {ProcessTranslationsResult}
  */
 export function processTranslationsFile(sourcePath: string, sourceContent?: string | undefined, options?: {
     locale?: string;
-} | undefined): import("../types").ProcessTranslationsResult;
+} | undefined): ProcessTranslationsResult;
 import { IntlCompiledMessageFormat } from "@discord/intl-message-database";
+import type { IntlMessagesFileDescriptor } from '../types';
+import type { IntlSourceFileInsertionData } from '../types';
+import type { ProcessDefinitionsResult } from '../types';
+import type { ProcessTranslationsResult } from '../types';

package/types/src/watcher.d.ts

@@ -3,7 +3,7 @@ export type BasicWatcher = {
 };
 /**
  * @typedef {{
- *   on(event: string, callback: (event: string, path: string) => void): void
+ *   on(event: string, callback: (event: string, path: string) => void): void,
  * }} BasicWatcher
  */
 /**

package/types/types.d.ts

@@ -1,4 +1,4 @@
-import { IntlSourceFile } from '@discord/intl-message-database';
+import { IntlSourceFile, IntlSourceFileError } from '@discord/intl-message-database';
 
 export type * from '@discord/intl-message-database';
 
@@ -72,11 +72,17 @@ export interface MessageDefinitionsTransformerOptions {
   bindMode?: 'proxy' | 'literal';
 }
 
-/**
- * The result of calling `processDefinitionsFile`, including the created source file, locale map,
- * and more.
- */
-export interface ProcessDefinitionsResult {
+export interface IntlProcessingError {
+  succeeded: false;
+  errors: IntlSourceFileError[];
+}
+
+export interface ProcessDefinitionsSuccess {
+  /**
+   * Whether the processing was successful as a whole. If any part of the process contained errors,
+   * including processing translations alongside the definitions file, this will be set to `false`.
+   */
+  succeeded: true;
   /**
    * Direct source file from the database that was created or updated by this process.
    */
@@ -108,10 +114,13 @@ export interface ProcessDefinitionsResult {
 }
 
 /**
- * The result of calling `processTranslationsFile`, including the created source file, locale map,
+ * The result of calling `processDefinitionsFile`, including the created source file, locale map,
  * and more.
  */
-export interface ProcessTranslationsResult {
+export type ProcessDefinitionsResult = ProcessDefinitionsSuccess | IntlProcessingError;
+
+export interface ProcessTranslationsSuccess {
+  succeeded: true;
   /**
    * Direct source file from the database that was created or updated by this process.
    */
@@ -128,3 +137,9 @@ export interface ProcessTranslationsResult {
    */
   locale: string;
 }
+
+/**
+ * The result of calling `processTranslationsFile`, including the created source file, locale map,
+ * and more.
+ */
+export type ProcessTranslationsResult = ProcessTranslationsSuccess | IntlProcessingError;

package/types.d.ts

@@ -1,4 +1,4 @@
-import { IntlSourceFile } from '@discord/intl-message-database';
+import { IntlSourceFile, IntlSourceFileError } from '@discord/intl-message-database';
 
 export type * from '@discord/intl-message-database';
 
@@ -72,11 +72,17 @@ export interface MessageDefinitionsTransformerOptions {
   bindMode?: 'proxy' | 'literal';
 }
 
-/**
- * The result of calling `processDefinitionsFile`, including the created source file, locale map,
- * and more.
- */
-export interface ProcessDefinitionsResult {
+export interface IntlProcessingError {
+  succeeded: false;
+  errors: IntlSourceFileError[];
+}
+
+export interface ProcessDefinitionsSuccess {
+  /**
+   * Whether the processing was successful as a whole. If any part of the process contained errors,
+   * including processing translations alongside the definitions file, this will be set to `false`.
+   */
+  succeeded: true;
   /**
    * Direct source file from the database that was created or updated by this process.
    */
@@ -108,10 +114,13 @@ export interface ProcessDefinitionsResult {
 }
 
 /**
- * The result of calling `processTranslationsFile`, including the created source file, locale map,
+ * The result of calling `processDefinitionsFile`, including the created source file, locale map,
  * and more.
  */
-export interface ProcessTranslationsResult {
+export type ProcessDefinitionsResult = ProcessDefinitionsSuccess | IntlProcessingError;
+
+export interface ProcessTranslationsSuccess {
+  succeeded: true;
   /**
    * Direct source file from the database that was created or updated by this process.
    */
@@ -128,3 +137,9 @@ export interface ProcessTranslationsResult {
    */
   locale: string;
 }
+
+/**
+ * The result of calling `processTranslationsFile`, including the created source file, locale map,
+ * and more.
+ */
+export type ProcessTranslationsResult = ProcessTranslationsSuccess | IntlProcessingError;