@discord/intl-loader-core 0.21.1 → 0.22.0-canary.558dd86

package/index.js

@@ -3,6 +3,7 @@ const {
   isMessageDefinitionsFile,
   isMessageTranslationsFile,
   IntlCompiledMessageFormat,
+  IntlDatabaseInsertStrategy,
 } = require('@discord/intl-message-database');
 
 const { database } = require('./src/database');
@@ -24,6 +25,9 @@ module.exports = {
   // @ts-expect-error This is a const enum, which TypeScript doesn't like letting you export, even
   // though it's a tangible object that can be accessed just fine from normal JS.
   IntlCompiledMessageFormat,
+  // @ts-expect-error This is a const enum, which TypeScript doesn't like letting you export, even
+  // though it's a tangible object that can be accessed just fine from normal JS.
+  IntlDatabaseInsertStrategy,
   MessageDefinitionsTransformer,
   database,
   findAllTranslationFiles,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@discord/intl-loader-core",
-  "version": "0.21.1",
+  "version": "0.22.0-canary.558dd86",
   "license": "MIT",
   "description": "Core utilities for writing loaders and transformers using @discord/intl",
   "author": "Jon Egeland",
@@ -28,7 +28,7 @@
   "dependencies": {
     "chokidar": "^3.6.0",
     "debug": "^4.3.6",
-    "@discord/intl-message-database": "0.21.1"
+    "@discord/intl-message-database": "0.22.0-rc.1"
   },
   "devDependencies": {
     "@types/debug": "^4.1.12",

package/src/processing.js

@@ -1,12 +1,22 @@
 const path = require('node:path');
 
 const debug = require('debug')('intl:loader-core');
-const { IntlCompiledMessageFormat } = require('@discord/intl-message-database');
+const {
+  IntlCompiledMessageFormat,
+  IntlDatabaseInsertStrategy,
+} = require('@discord/intl-message-database');
 
 const { database } = require('./database');
 const { findAllTranslationFiles, getLocaleFromTranslationsFileName } = require('./util');
 
 /**
+ * @import {
+ *   IntlMessagesFileDescriptor,
+ *   IntlSourceFileInsertionData,
+ *   ProcessDefinitionsResult,
+ *   ProcessTranslationsResult
+ * } from '../types';
+ *
  * @typedef {{
  *   format?: IntlCompiledMessageFormat,
  *   bundleSecrets?: boolean,
@@ -63,7 +73,6 @@ function findAllDefinitionsFilesForTranslations(filePath) {
   for (const file of sourceFiles) {
     const source = database.getSourceFile(file);
     if (source.type === 'definition' && source.meta.translationsPath === expectedTranslationsPath) {
-      console.log(file, source.meta);
       relevantPaths.push(source.file);
     }
   }
@@ -89,7 +98,7 @@ function findAllMessagesFiles(directories, defaultLocale = 'en-US') {
  *
  * @param {string[]} files
  * @param {string} defaultLocale
- * @returns {import('../types').IntlMessagesFileDescriptor[]}
+ * @returns {IntlMessagesFileDescriptor[]}
  */
 function filterAllMessagesFiles(files, defaultLocale = 'en-US') {
   return database.filterAllMessagesFiles(files, defaultLocale);
@@ -97,17 +106,19 @@ function filterAllMessagesFiles(files, defaultLocale = 'en-US') {
 
 /**
  * Iterate the given `files`, processing each one's content into the database. Processing is done
- * in parallel using native acceleration, and returns the list of all file names that were used.
+ * in parallel using native acceleration, and returns an object describing all of the changes that
+ * were processed.
  *
  * Note that this method _only_ operates by reading file contents from the system, it is not
  * possible to supply preloaded content through a Buffer in the same way as `processDefinitionsFile`
  * with a `sourceContent` argument.
  *
- * @param {import('../types').IntlMessagesFileDescriptor[]} files
- * @returns {import('../types').IntlMultiProcessingResult}
+ * @param {IntlMessagesFileDescriptor[]} files
+ * @param {IntlDatabaseInsertStrategy} [strategy]
+ * @returns {IntlSourceFileInsertionData[]}
  */
-function processAllMessagesFiles(files) {
-  return database.processAllMessagesFiles(files);
+function processAllMessagesFiles(files, strategy = IntlDatabaseInsertStrategy.Update) {
+  return database.processAllMessagesFiles(files, strategy);
 }
 
 /**
@@ -116,21 +127,28 @@ function processAllMessagesFiles(files) {
  * @param {{
  *   processTranslations?: boolean,
  *   locale?: string
+ *   strategy?: IntlDatabaseInsertStrategy
  * }=} options
- * @returns {import('../types').ProcessDefinitionsResult}
+ * @returns {ProcessDefinitionsResult}
  */
 function processDefinitionsFile(sourcePath, sourceContent, options = {}) {
   const {
     processTranslations = false,
     // TODO: Make this more configurable/automatically determined.
     locale = 'en-US',
+    strategy = IntlDatabaseInsertStrategy.Update,
   } = options;
   debug(`[${sourcePath}] Processing definitions with locale "${locale}"`);
 
+  /** @type {IntlSourceFileInsertionData} */
+  let result;
   if (sourceContent != null) {
-    database.processDefinitionsFileContent(sourcePath, sourceContent, locale);
+    result = database.processDefinitionsFileContent(sourcePath, sourceContent, locale, strategy);
   } else {
-    database.processDefinitionsFile(sourcePath, locale);
+    result = database.processDefinitionsFile(sourcePath, locale, strategy);
+  }
+  if (result.errors.length > 0) {
+    return { succeeded: false, errors: result.errors };
   }
 
   const sourceFile = database.getSourceFile(sourcePath);
@@ -150,11 +168,16 @@ function processDefinitionsFile(sourcePath, sourceContent, options = {}) {
   );
 
   if (processTranslations) {
-    const translationsResult = database.processAllTranslationFiles(translationsLocaleMap);
+    const translationsResult = database.processAllTranslationFiles(translationsLocaleMap, strategy);
+    const allErrors = translationsResult.flatMap((result) => result.errors);
+    if (allErrors.length > 0) {
+      return { succeeded: false, errors: allErrors };
+    }
     debug('[${sourcePath}] Finished processing all translations: %O', translationsResult);
   }
 
   return {
+    succeeded: true,
     sourceFile,
     locale,
     messageKeys,
@@ -169,15 +192,25 @@ function processDefinitionsFile(sourcePath, sourceContent, options = {}) {
  * @param {string=} sourceContent
  * @param {{
  *   locale?: string,
+ *   strategy?: IntlDatabaseInsertStrategy
  * }=} options
- * @returns {import('../types').ProcessTranslationsResult}
+ * @returns {ProcessTranslationsResult}
  */
 function processTranslationsFile(sourcePath, sourceContent, options = {}) {
-  const { locale = getLocaleFromTranslationsFileName(sourcePath) } = options;
+  const {
+    locale = getLocaleFromTranslationsFileName(sourcePath),
+    strategy = IntlDatabaseInsertStrategy.Update,
+  } = options;
+
+  /** @type {IntlSourceFileInsertionData} */
+  let result;
   if (sourceContent) {
-    database.processTranslationFileContent(sourcePath, locale, sourceContent);
+    result = database.processTranslationFileContent(sourcePath, locale, sourceContent, strategy);
   } else {
-    database.processTranslationFile(sourcePath, locale);
+    result = database.processTranslationFile(sourcePath, locale, strategy);
+  }
+  if (result.errors.length > 0) {
+    return { succeeded: false, errors: result.errors };
   }
 
   const sourceFile = database.getSourceFile(sourcePath);
@@ -189,6 +222,7 @@ function processTranslationsFile(sourcePath, sourceContent, options = {}) {
   }
 
   return {
+    succeeded: true,
     sourceFile,
     locale,
     messageKeys: database.getSourceFileKeyMap(sourcePath),

package/src/watcher.js

@@ -7,7 +7,6 @@ const {
   IntlCompiledMessageFormat,
 } = require('@discord/intl-message-database');
 
-const { database } = require('./database');
 const {
   processDefinitionsFile,
   precompileFileForLocale,
@@ -30,7 +29,7 @@ const DEFAULT_LOCALE = 'en-US';
 
 /**
  * @typedef {{
- *   on(event: string, callback: (event: string, path: string) => void): void
+ *   on(event: string, callback: (event: string, path: string) => void): void,
  * }} BasicWatcher
  */
 
@@ -53,7 +52,11 @@ async function watchMessagesFiles(
 ) {
   debug('Pre-initializing database with all discoverable messages files');
   const files = findAllMessagesFiles(watchedFolders);
-  processAllMessagesFiles(files);
+  const result = processAllMessagesFiles(files);
+  const reprocessQueue = result
+    .filter((data) => data.errors.length > 0)
+    .map((data) => data.fileKey);
+
   if (once) {
     debug('Performing one-shot watch because `once` was true');
     // This is a weird hack around file watching just not working well on
@@ -80,8 +83,27 @@ async function watchMessagesFiles(
     ignoreInitial: skipInitial,
     persistent: !once,
   });
+
   watcher.on('all', (event, path) => {
-    processAllMessagesFiles(filterAllMessagesFiles([path]));
+    const files = filterAllMessagesFiles([path]);
+    if (files.length === 0) return;
+
+    // Process the file that changed first, then reprocess all of the queued files from previous
+    // failures to hopefully get them to process successfully again.
+    const [result] = processAllMessagesFiles(files);
+    const failed = result.errors.length > 0 ? [result.fileKey] : [];
+    const secondResult = processAllMessagesFiles(
+      filterAllMessagesFiles([...reprocessQueue, ...failed]),
+    );
+    // For any files that are still failing, keep them in the queue for next time.
+    const secondFailures = secondResult
+      .filter((data) => data.errors.length > 0)
+      .map((data) => data.fileKey);
+    if (secondFailures.length > 0) {
+      debug('Failed to re-process files: %O. Next update will try again.', secondFailures);
+    }
+    reprocessQueue.splice(0, Infinity, ...secondFailures);
+
     debug(`watcher got event: ${event}, for path ${path}`);
   });
   for (const signal of ['SIGINT', 'SIGTERM', 'SIGQUIT']) {
@@ -136,16 +158,17 @@ async function precompileMessageDefinitionsFiles(
         `.compiled.messages.${assetExtension}`,
       );
       const result = processDefinitionsFile(filePath);
-      precompileFileForLocale(filePath, result.locale, undefined, {
+      if (!result.succeeded) {
+        debug('[INTL Error] Failed to compile messages');
+        console.error(result.errors);
+        return;
+      }
+
+      precompileFileForLocale(filePath, result.locale, outputPath, {
         format,
         bundleSecrets,
       });
 
-      database.processDefinitionsFile(filePath);
-      database.precompile(filePath, DEFAULT_LOCALE, outputPath, {
-        format: IntlCompiledMessageFormat.KeylessJson,
-        bundleSecrets,
-      });
       debug(`Wrote definitions to: ${outputPath}`);
     } catch (e) {
       debug('[INTL Error] Failed to compile messages');

package/types/index.d.ts

@@ -1,4 +1,5 @@
 import { IntlCompiledMessageFormat } from "@discord/intl-message-database";
+import { IntlDatabaseInsertStrategy } from "@discord/intl-message-database";
 import { MessageDefinitionsTransformer } from "./src/transformer";
 import { database } from "./src/database";
 import { findAllTranslationFiles } from "./src/util";
@@ -15,4 +16,4 @@ import { findAllMessagesFiles } from "./src/processing";
 import { filterAllMessagesFiles } from "./src/processing";
 import { processAllMessagesFiles } from "./src/processing";
 import watcher = require("./src/watcher");
-export { IntlCompiledMessageFormat, MessageDefinitionsTransformer, database, findAllTranslationFiles, findAllDefinitionsFilesForTranslations, getLocaleFromTranslationsFileName, generateTypeDefinitions, hashMessageKey, isMessageDefinitionsFile, isMessageTranslationsFile, processDefinitionsFile, processTranslationsFile, precompileFileForLocale, findAllMessagesFiles, filterAllMessagesFiles, processAllMessagesFiles, watcher };
+export { IntlCompiledMessageFormat, IntlDatabaseInsertStrategy, MessageDefinitionsTransformer, database, findAllTranslationFiles, findAllDefinitionsFilesForTranslations, getLocaleFromTranslationsFileName, generateTypeDefinitions, hashMessageKey, isMessageDefinitionsFile, isMessageTranslationsFile, processDefinitionsFile, processTranslationsFile, precompileFileForLocale, findAllMessagesFiles, filterAllMessagesFiles, processAllMessagesFiles, watcher };

package/types/src/processing.d.ts

@@ -32,21 +32,23 @@ export function findAllMessagesFiles(directories: string[], defaultLocale?: stri
  *
  * @param {string[]} files
  * @param {string} defaultLocale
- * @returns {import('../types').IntlMessagesFileDescriptor[]}
+ * @returns {IntlMessagesFileDescriptor[]}
  */
-export function filterAllMessagesFiles(files: string[], defaultLocale?: string): import("../types").IntlMessagesFileDescriptor[];
+export function filterAllMessagesFiles(files: string[], defaultLocale?: string): IntlMessagesFileDescriptor[];
 /**
  * Iterate the given `files`, processing each one's content into the database. Processing is done
- * in parallel using native acceleration, and returns the list of all file names that were used.
+ * in parallel using native acceleration, and returns an object describing all of the changes that
+ * were processed.
  *
  * Note that this method _only_ operates by reading file contents from the system, it is not
  * possible to supply preloaded content through a Buffer in the same way as `processDefinitionsFile`
  * with a `sourceContent` argument.
  *
- * @param {import('../types').IntlMessagesFileDescriptor[]} files
- * @returns {import('../types').IntlMultiProcessingResult}
+ * @param {IntlMessagesFileDescriptor[]} files
+ * @param {IntlDatabaseInsertStrategy} [strategy]
+ * @returns {IntlSourceFileInsertionData[]}
  */
-export function processAllMessagesFiles(files: import("../types").IntlMessagesFileDescriptor[]): import("../types").IntlMultiProcessingResult;
+export function processAllMessagesFiles(files: IntlMessagesFileDescriptor[], strategy?: IntlDatabaseInsertStrategy | undefined): IntlSourceFileInsertionData[];
 /**
  * Generate a `.d.ts` file containing TypeScript type definitions for all of the messages defined in
  * `sourcePath`. This method does not process `sourcePath` at all, meaning it expects the database
@@ -92,23 +94,32 @@ export function precompileFileForLocale(sourcePath: string, locale: string, outp
  * @param {{
  *   processTranslations?: boolean,
  *   locale?: string
+ *   strategy?: IntlDatabaseInsertStrategy
  * }=} options
- * @returns {import('../types').ProcessDefinitionsResult}
+ * @returns {ProcessDefinitionsResult}
  */
 export function processDefinitionsFile(sourcePath: string, sourceContent?: string | undefined, options?: {
     processTranslations?: boolean;
     locale?: string;
-} | undefined): import("../types").ProcessDefinitionsResult;
+    strategy?: IntlDatabaseInsertStrategy;
+} | undefined): ProcessDefinitionsResult;
 /**
  *
  * @param {string} sourcePath
  * @param {string=} sourceContent
  * @param {{
  *   locale?: string,
+ *   strategy?: IntlDatabaseInsertStrategy
  * }=} options
- * @returns {import('../types').ProcessTranslationsResult}
+ * @returns {ProcessTranslationsResult}
  */
 export function processTranslationsFile(sourcePath: string, sourceContent?: string | undefined, options?: {
     locale?: string;
-} | undefined): import("../types").ProcessTranslationsResult;
+    strategy?: IntlDatabaseInsertStrategy;
+} | undefined): ProcessTranslationsResult;
 import { IntlCompiledMessageFormat } from "@discord/intl-message-database";
+import type { IntlMessagesFileDescriptor } from '../types';
+import { IntlDatabaseInsertStrategy } from "@discord/intl-message-database";
+import type { IntlSourceFileInsertionData } from '../types';
+import type { ProcessDefinitionsResult } from '../types';
+import type { ProcessTranslationsResult } from '../types';

package/types/src/watcher.d.ts

@@ -3,7 +3,7 @@ export type BasicWatcher = {
 };
 /**
  * @typedef {{
- *   on(event: string, callback: (event: string, path: string) => void): void
+ *   on(event: string, callback: (event: string, path: string) => void): void,
  * }} BasicWatcher
  */
 /**

package/types/types.d.ts

@@ -1,4 +1,4 @@
-import { IntlSourceFile } from '@discord/intl-message-database';
+import { IntlSourceFile, IntlSourceFileError } from '@discord/intl-message-database';
 
 export type * from '@discord/intl-message-database';
 
@@ -72,11 +72,17 @@ export interface MessageDefinitionsTransformerOptions {
   bindMode?: 'proxy' | 'literal';
 }
 
-/**
- * The result of calling `processDefinitionsFile`, including the created source file, locale map,
- * and more.
- */
-export interface ProcessDefinitionsResult {
+export interface IntlProcessingError {
+  succeeded: false;
+  errors: IntlSourceFileError[];
+}
+
+export interface ProcessDefinitionsSuccess {
+  /**
+   * Whether the processing was successful as a whole. If any part of the process contained errors,
+   * including processing translations alongside the definitions file, this will be set to `false`.
+   */
+  succeeded: true;
   /**
    * Direct source file from the database that was created or updated by this process.
    */
@@ -108,10 +114,13 @@ export interface ProcessDefinitionsResult {
 }
 
 /**
- * The result of calling `processTranslationsFile`, including the created source file, locale map,
+ * The result of calling `processDefinitionsFile`, including the created source file, locale map,
  * and more.
  */
-export interface ProcessTranslationsResult {
+export type ProcessDefinitionsResult = ProcessDefinitionsSuccess | IntlProcessingError;
+
+export interface ProcessTranslationsSuccess {
+  succeeded: true;
   /**
    * Direct source file from the database that was created or updated by this process.
    */
@@ -128,3 +137,9 @@ export interface ProcessTranslationsResult {
    */
   locale: string;
 }
+
+/**
+ * The result of calling `processTranslationsFile`, including the created source file, locale map,
+ * and more.
+ */
+export type ProcessTranslationsResult = ProcessTranslationsSuccess | IntlProcessingError;

package/types.d.ts

@@ -1,4 +1,4 @@
-import { IntlSourceFile } from '@discord/intl-message-database';
+import { IntlSourceFile, IntlSourceFileError } from '@discord/intl-message-database';
 
 export type * from '@discord/intl-message-database';
 
@@ -72,11 +72,17 @@ export interface MessageDefinitionsTransformerOptions {
   bindMode?: 'proxy' | 'literal';
 }
 
-/**
- * The result of calling `processDefinitionsFile`, including the created source file, locale map,
- * and more.
- */
-export interface ProcessDefinitionsResult {
+export interface IntlProcessingError {
+  succeeded: false;
+  errors: IntlSourceFileError[];
+}
+
+export interface ProcessDefinitionsSuccess {
+  /**
+   * Whether the processing was successful as a whole. If any part of the process contained errors,
+   * including processing translations alongside the definitions file, this will be set to `false`.
+   */
+  succeeded: true;
   /**
    * Direct source file from the database that was created or updated by this process.
    */
@@ -108,10 +114,13 @@ export interface ProcessDefinitionsResult {
 }
 
 /**
- * The result of calling `processTranslationsFile`, including the created source file, locale map,
+ * The result of calling `processDefinitionsFile`, including the created source file, locale map,
  * and more.
  */
-export interface ProcessTranslationsResult {
+export type ProcessDefinitionsResult = ProcessDefinitionsSuccess | IntlProcessingError;
+
+export interface ProcessTranslationsSuccess {
+  succeeded: true;
   /**
    * Direct source file from the database that was created or updated by this process.
    */
@@ -128,3 +137,9 @@ export interface ProcessTranslationsResult {
    */
   locale: string;
 }
+
+/**
+ * The result of calling `processTranslationsFile`, including the created source file, locale map,
+ * and more.
+ */
+export type ProcessTranslationsResult = ProcessTranslationsSuccess | IntlProcessingError;