next-intl 4.5.8 → 4.6.1

package/dist/esm/development/extractor/catalog/CatalogManager.js

@@ -1,19 +1,25 @@
 import fs from 'fs/promises';
 import path from 'path';
-import MessageExtractor from '../extractor/MessageExtractor.js';
-import formatters from '../formatters/index.js';
+import { resolveCodec, getFormatExtension } from '../format/index.js';
 import SourceFileScanner from '../source/SourceFileScanner.js';
-import { localeCompare } from '../utils.js';
+import SourceFileWatcher from '../source/SourceFileWatcher.js';
+import { getDefaultProjectRoot, localeCompare } from '../utils.js';
 import CatalogLocales from './CatalogLocales.js';
 import CatalogPersister from './CatalogPersister.js';
 import SaveScheduler from './SaveScheduler.js';
 
 class CatalogManager {
-  /* The source of truth for which messages are used. */
+  /**
+   * The source of truth for which messages are used.
+   * NOTE: Should be mutated in place to keep `messagesById` and `messagesByFile` in sync.
+   */
   messagesByFile = (() => new Map())();
 
-  /* Fast lookup for messages by ID across all files,
-   * contains the same messages as `messagesByFile`. */
+  /**
+   * Fast lookup for messages by ID across all files,
+   * contains the same messages as `messagesByFile`.
+   * NOTE: Should be mutated in place to keep `messagesById` and `messagesByFile` in sync.
+   */
   messagesById = (() => new Map())();
 
   /**
@@ -26,104 +32,92 @@ class CatalogManager {
   // Cached instances
 
   // Resolves when all catalogs are loaded
-  // (but doesn't indicate that project scan is done)
 
-  constructor(config, opts = {}) {
+  // Resolves when the initial project scan and processing is complete
+
+  constructor(config, opts) {
     this.config = config;
     this.saveScheduler = new SaveScheduler(50);
-    this.projectRoot = opts.projectRoot || process.cwd();
+    this.projectRoot = opts.projectRoot ?? getDefaultProjectRoot();
     this.isDevelopment = opts.isDevelopment ?? false;
-    this.messageExtractor = new MessageExtractor({
-      isDevelopment: this.isDevelopment,
-      projectRoot: this.projectRoot,
-      sourceMap: opts.sourceMap
-    });
+    this.extractor = opts.extractor;
+    if (this.isDevelopment) {
+      // We kick this off as early as possible, so we get notified about changes
+      // that happen during the initial project scan (while awaiting it to
+      // complete though)
+      this.sourceWatcher = new SourceFileWatcher(this.getSrcPaths(), this.handleFileEvents.bind(this));
+      void this.sourceWatcher.start();
+    }
   }
-  async getFormatter() {
-    if (this.formatter) {
-      return this.formatter;
-    } else {
-      const FormatterClass = (await formatters[this.config.messages.format]()).default;
-      this.formatter = new FormatterClass();
-      return this.formatter;
+  async getCodec() {
+    if (!this.codec) {
+      this.codec = await resolveCodec(this.config.messages.format, this.projectRoot);
     }
+    return this.codec;
   }
   async getPersister() {
     if (this.persister) {
       return this.persister;
     } else {
-      this.persister = new CatalogPersister(this.config.messages.path, await this.getFormatter());
+      this.persister = new CatalogPersister({
+        messagesPath: this.config.messages.path,
+        codec: await this.getCodec(),
+        extension: getFormatExtension(this.config.messages.format)
+      });
       return this.persister;
     }
   }
-  async getCatalogLocales() {
+  getCatalogLocales() {
     if (this.catalogLocales) {
       return this.catalogLocales;
     } else {
       const messagesDir = path.join(this.projectRoot, this.config.messages.path);
-      const formatter = await this.getFormatter();
       this.catalogLocales = new CatalogLocales({
         messagesDir,
         sourceLocale: this.config.sourceLocale,
-        extension: formatter.EXTENSION,
+        extension: getFormatExtension(this.config.messages.format),
         locales: this.config.messages.locales
       });
       return this.catalogLocales;
     }
   }
   async getTargetLocales() {
-    const catalogLocales = await this.getCatalogLocales();
-    return catalogLocales.getTargetLocales();
+    return this.getCatalogLocales().getTargetLocales();
   }
   getSrcPaths() {
     return (Array.isArray(this.config.srcPath) ? this.config.srcPath : [this.config.srcPath]).map(srcPath => path.join(this.projectRoot, srcPath));
   }
   async loadMessages() {
-    this.loadCatalogsPromise = Promise.all([this.loadSourceMessages(), this.loadTargetMessages()]);
-
-    // Ensure catalogs are loaded before scanning source files.
-    // Otherwise, `loadSourceMessages` might overwrite extracted
-    // messages if it finishes after source file extraction.
+    const sourceDiskMessages = await this.loadSourceMessages();
+    this.loadCatalogsPromise = this.loadTargetMessages();
     await this.loadCatalogsPromise;
+    this.scanCompletePromise = (async () => {
+      const sourceFiles = await SourceFileScanner.getSourceFiles(this.getSrcPaths());
+      await Promise.all(Array.from(sourceFiles).map(async filePath => this.processFile(filePath)));
+      this.mergeSourceDiskMetadata(sourceDiskMessages);
+    })();
+    await this.scanCompletePromise;
     if (this.isDevelopment) {
-      const catalogLocales = await this.getCatalogLocales();
+      const catalogLocales = this.getCatalogLocales();
       catalogLocales.subscribeLocalesChange(this.onLocalesChange);
     }
-    const sourceFiles = await SourceFileScanner.getSourceFiles(this.getSrcPaths());
-    await Promise.all(sourceFiles.map(async filePath => this.extractFileMessages(filePath, await fs.readFile(filePath, 'utf8'))));
   }
   async loadSourceMessages() {
-    // First hydrate from source locale file to potentially init metadata
-    const messages = await this.loadLocaleMessages(this.config.sourceLocale);
-    const messagesById = new Map();
-    const messagesByFile = new Map();
-    for (const message of messages) {
-      messagesById.set(message.id, message);
-      if (message.references) {
-        for (const ref of message.references) {
-          const absoluteFilePath = path.join(this.projectRoot, ref.path);
-          let fileMessages = messagesByFile.get(absoluteFilePath);
-          if (!fileMessages) {
-            fileMessages = new Map();
-            messagesByFile.set(absoluteFilePath, fileMessages);
-          }
-          fileMessages.set(message.id, message);
-        }
-      }
+    // Load source catalog to hydrate metadata (e.g. flags) later without
+    // treating catalog entries as source of truth.
+    const diskMessages = await this.loadLocaleMessages(this.config.sourceLocale);
+    const byId = new Map();
+    for (const diskMessage of diskMessages) {
+      byId.set(diskMessage.id, diskMessage);
     }
-    this.messagesById = messagesById;
-    this.messagesByFile = messagesByFile;
+    return byId;
   }
   async loadLocaleMessages(locale) {
     const persister = await this.getPersister();
-    try {
-      const messages = await persister.read(locale);
-      const fileTime = await persister.getLastModified(locale);
-      this.lastWriteByLocale.set(locale, fileTime);
-      return messages;
-    } catch {
-      return [];
-    }
+    const messages = await persister.read(locale);
+    const fileTime = await persister.getLastModified(locale);
+    this.lastWriteByLocale.set(locale, fileTime);
+    return messages;
   }
   async loadTargetMessages() {
     const targetLocales = await this.getTargetLocales();
@@ -136,28 +130,70 @@ class CatalogManager {
       for (const diskMessage of diskMessages) {
         const prev = this.messagesById.get(diskMessage.id);
         if (prev) {
+          // Mutate the existing object instead of creating a copy
+          // to keep messagesById and messagesByFile in sync.
           // Unknown properties (like flags): disk wins
           // Known properties: existing (from extraction) wins
-          this.messagesById.set(diskMessage.id, {
-            ...diskMessage,
-            id: prev.id,
-            message: prev.message,
-            description: prev.description,
-            references: prev.references
-          });
+          for (const key of Object.keys(diskMessage)) {
+            if (!['id', 'message', 'description', 'references'].includes(key)) {
+              // For unknown properties (like flags), disk wins
+              prev[key] = diskMessage[key];
+            }
+          }
         }
       }
     } else {
-      // For target: disk wins completely
-      const translations = new Map();
-      for (const message of diskMessages) {
-        translations.set(message.id, message);
+      // For target: disk wins completely, BUT preserve existing translations
+      // if we read empty (likely a write in progress by an external tool
+      // that causes the file to temporarily be empty)
+      const existingTranslations = this.translationsByTargetLocale.get(locale);
+      const hasExistingTranslations = existingTranslations && existingTranslations.size > 0;
+      if (diskMessages.length > 0) {
+        // We got content from disk, replace with it
+        const translations = new Map();
+        for (const message of diskMessages) {
+          translations.set(message.id, message);
+        }
+        this.translationsByTargetLocale.set(locale, translations);
+      } else if (hasExistingTranslations) ; else {
+        // We read empty and have no existing translations
+        const translations = new Map();
+        this.translationsByTargetLocale.set(locale, translations);
+      }
+    }
+  }
+  mergeSourceDiskMetadata(diskMessages) {
+    for (const [id, diskMessage] of diskMessages) {
+      const existing = this.messagesById.get(id);
+      if (!existing) continue;
+
+      // Mutate the existing object instead of creating a copy.
+      // This keeps `messagesById` and `messagesByFile` in sync since
+      // they reference the same object instance.
+      for (const key of Object.keys(diskMessage)) {
+        if (existing[key] == null) {
+          existing[key] = diskMessage[key];
+        }
       }
-      this.translationsByTargetLocale.set(locale, translations);
     }
   }
-  async extractFileMessages(absoluteFilePath, source) {
-    const result = await this.messageExtractor.processFileContent(absoluteFilePath, source);
+  async processFile(absoluteFilePath) {
+    let messages = [];
+    try {
+      const content = await fs.readFile(absoluteFilePath, 'utf8');
+      let extraction;
+      try {
+        extraction = await this.extractor.extract(absoluteFilePath, content);
+      } catch {
+        return false;
+      }
+      messages = extraction.messages;
+    } catch (err) {
+      if (err.code !== 'ENOENT') {
+        throw err;
+      }
+      // ENOENT -> treat as no messages
+    }
     const prevFileMessages = this.messagesByFile.get(absoluteFilePath);
 
     // Init with all previous ones
@@ -165,24 +201,17 @@ class CatalogManager {
 
     // Replace existing messages with new ones
     const fileMessages = new Map();
-    for (let message of result.messages) {
+    for (let message of messages) {
       const prevMessage = this.messagesById.get(message.id);
 
       // Merge with previous message if it exists
       if (prevMessage) {
-        // References: The `message` we receive here will always have one
-        // reference, which is the current file. We need to merge this with
-        // potentially existing references.
-        const references = [...(prevMessage.references ?? [])];
-        message.references.forEach(ref => {
-          if (!references.some(cur => cur.path === ref.path)) {
-            references.push(ref);
-          }
-        });
-        references.sort((referenceA, referenceB) => localeCompare(referenceA.path, referenceB.path));
+        const validated = prevMessage.references ?? [];
         message = {
           ...message,
-          references
+          references: this.mergeReferences(validated, {
+            path: path.relative(this.projectRoot, absoluteFilePath)
+          })
         };
 
         // Merge other properties like description, or unknown
@@ -200,31 +229,39 @@ class CatalogManager {
       const index = idsToRemove.indexOf(message.id);
       if (index !== -1) idsToRemove.splice(index, 1);
     }
-
-    // Don't delete IDs still used in other files
     const relativeFilePath = path.relative(this.projectRoot, absoluteFilePath);
-    const idsToDelete = idsToRemove.filter(id => {
-      const message = this.messagesById.get(id);
-      return !message?.references?.some(ref => ref.path !== relativeFilePath);
-    });
 
     // Clean up removed messages from `messagesById`
-    idsToDelete.forEach(id => {
-      this.messagesById.delete(id);
+    idsToRemove.forEach(id => {
+      const message = this.messagesById.get(id);
+      if (!message) return;
+      const hasOtherReferences = message.references?.some(ref => ref.path !== relativeFilePath);
+      if (!hasOtherReferences) {
+        // No other references, delete the message entirely
+        this.messagesById.delete(id);
+      } else {
+        // Message is used elsewhere, remove this file from references
+        // Mutate the existing object to keep `messagesById` and `messagesByFile` in sync
+        message.references = message.references?.filter(ref => ref.path !== relativeFilePath);
+      }
     });
 
     // Update the stored messages
-    const hasMessages = result.messages.length > 0;
-    if (hasMessages) {
+    if (messages.length > 0) {
       this.messagesByFile.set(absoluteFilePath, fileMessages);
     } else {
       this.messagesByFile.delete(absoluteFilePath);
     }
     const changed = this.haveMessagesChangedForFile(prevFileMessages, fileMessages);
-    return {
-      ...result,
-      changed
-    };
+    return changed;
+  }
+  mergeReferences(existing, current) {
+    const dedup = new Map();
+    for (const ref of existing) {
+      dedup.set(ref.path, ref);
+    }
+    dedup.set(current.path, current);
+    return Array.from(dedup.values()).sort((a, b) => localeCompare(a.path, b.path));
   }
   haveMessagesChangedForFile(beforeMessages, afterMessages) {
     // If one exists and the other doesn't, there's a change
@@ -273,18 +310,21 @@ class CatalogManager {
     if (currentFileTime && lastWriteTime && currentFileTime > lastWriteTime) {
       await this.reloadLocaleCatalog(locale);
     }
-    const prevMessages = isSourceLocale ? this.messagesById : this.translationsByTargetLocale.get(locale);
-    const localeMessages = messages.map(message => {
-      const prev = prevMessages?.get(message.id);
+    const localeMessages = isSourceLocale ? this.messagesById : this.translationsByTargetLocale.get(locale);
+    const messagesToPersist = messages.map(message => {
+      const localeMessage = localeMessages?.get(message.id);
       return {
-        ...prev,
+        ...localeMessage,
         id: message.id,
         description: message.description,
         references: message.references,
-        message: isSourceLocale ? message.message : prev?.message ?? ''
+        message: isSourceLocale ? message.message : localeMessage?.message ?? ''
       };
     });
-    await persister.write(locale, localeMessages);
+    await persister.write(messagesToPersist, {
+      locale,
+      sourceMessagesById: this.messagesById
+    });
 
     // Update timestamps
     const newTime = await persister.getLastModified(locale);
@@ -301,8 +341,29 @@ class CatalogManager {
       this.lastWriteByLocale.delete(locale);
     }
   };
-  destroy() {
-    this.saveScheduler.destroy();
+  async handleFileEvents(events) {
+    if (this.loadCatalogsPromise) {
+      await this.loadCatalogsPromise;
+    }
+
+    // Wait for initial scan to complete to avoid race conditions
+    if (this.scanCompletePromise) {
+      await this.scanCompletePromise;
+    }
+    let changed = false;
+    const expandedEvents = await this.sourceWatcher.expandDirectoryDeleteEvents(events, Array.from(this.messagesByFile.keys()));
+    for (const event of expandedEvents) {
+      const hasChanged = await this.processFile(event.path);
+      changed ||= hasChanged;
+    }
+    if (changed) {
+      await this.save();
+    }
+  }
+  [Symbol.dispose]() {
+    this.sourceWatcher?.stop();
+    this.sourceWatcher = undefined;
+    this.saveScheduler[Symbol.dispose]();
     if (this.catalogLocales && this.isDevelopment) {
       this.catalogLocales.unsubscribeLocalesChange(this.onLocalesChange);
     }

package/dist/esm/development/extractor/catalog/CatalogPersister.js

@@ -2,25 +2,43 @@ import fs from 'fs/promises';
 import path from 'path';
 
 class CatalogPersister {
-  constructor(messagesPath, formatter) {
-    this.messagesPath = messagesPath;
-    this.formatter = formatter;
+  constructor(params) {
+    this.messagesPath = params.messagesPath;
+    this.codec = params.codec;
+    this.extension = params.extension;
+  }
+  getFileName(locale) {
+    return locale + this.extension;
   }
   getFilePath(locale) {
-    return path.join(this.messagesPath, locale + this.formatter.EXTENSION);
+    return path.join(this.messagesPath, this.getFileName(locale));
   }
   async read(locale) {
     const filePath = this.getFilePath(locale);
-    const content = await fs.readFile(filePath, 'utf8');
-    return this.formatter.parse(content, {
-      locale
-    });
+    let content;
+    try {
+      content = await fs.readFile(filePath, 'utf8');
+    } catch (error) {
+      if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') {
+        return [];
+      }
+      throw new Error(`Error while reading ${this.getFileName(locale)}:\n> ${error}`, {
+        cause: error
+      });
+    }
+    try {
+      return this.codec.decode(content, {
+        locale
+      });
+    } catch (error) {
+      throw new Error(`Error while decoding ${this.getFileName(locale)}:\n> ${error}`, {
+        cause: error
+      });
+    }
   }
-  async write(locale, messages) {
-    const filePath = this.getFilePath(locale);
-    const content = this.formatter.serialize(messages, {
-      locale
-    });
+  async write(messages, context) {
+    const filePath = this.getFilePath(context.locale);
+    const content = this.codec.encode(messages, context);
     try {
       const outputDir = path.dirname(filePath);
       await fs.mkdir(outputDir, {

package/dist/esm/development/extractor/catalog/SaveScheduler.js

@@ -71,7 +71,7 @@ class SaveScheduler {
       }
     }
   }
-  destroy() {
+  [Symbol.dispose]() {
     if (this.saveTimeout) {
       clearTimeout(this.saveTimeout);
       this.saveTimeout = undefined;

package/dist/esm/development/extractor/catalogLoader.js

@@ -1,13 +1,12 @@
 import path from 'path';
-import formatters from './formatters/index.js';
+import { getFormatExtension, resolveCodec } from './format/index.js';
 
-let cachedFormatter = null;
-async function getFormatter(options) {
-  if (!cachedFormatter) {
-    const FormatterClass = (await formatters[options.messages.format]()).default;
-    cachedFormatter = new FormatterClass();
+let cachedCodec = null;
+async function getCodec(options, projectRoot) {
+  if (!cachedCodec) {
+    cachedCodec = await resolveCodec(options.messages.format, projectRoot);
   }
-  return cachedFormatter;
+  return cachedCodec;
 }
 
 /**
@@ -20,9 +19,10 @@ async function getFormatter(options) {
 function catalogLoader(source) {
   const options = this.getOptions();
   const callback = this.async();
-  getFormatter(options).then(formatter => {
-    const locale = path.basename(this.resourcePath, formatter.EXTENSION);
-    const jsonString = formatter.toJSONString(source, {
+  const extension = getFormatExtension(options.messages.format);
+  getCodec(options, this.rootContext).then(codec => {
+    const locale = path.basename(this.resourcePath, extension);
+    const jsonString = codec.toJSONString(source, {
       locale
     });
 

package/dist/esm/development/extractor/extractMessages.js

@@ -1,8 +1,15 @@
 import ExtractionCompiler from './ExtractionCompiler.js';
+import MessageExtractor from './extractor/MessageExtractor.js';
+import { getDefaultProjectRoot } from './utils.js';
 
 async function extractMessages(params) {
-  const compiler = new ExtractionCompiler(params);
-  await compiler.extract();
+  const compiler = new ExtractionCompiler(params, {
+    extractor: new MessageExtractor({
+      isDevelopment: false,
+      projectRoot: getDefaultProjectRoot()
+    })
+  });
+  await compiler.extractAll();
 }
 
 export { extractMessages as default };

package/dist/esm/development/extractor/extractionLoader.js

@@ -1,21 +1,24 @@
-import ExtractionCompiler from './ExtractionCompiler.js';
+import MessageExtractor from './extractor/MessageExtractor.js';
 
-// This instance:
-// - Remains available through HMR
-// - Is the same across react-client and react-server
-// - Is only lost when the dev server restarts (e.g. due to change to Next.js config)
-let compiler;
+// Module-level extractor instance for transformation caching.
+// Note: Next.js/Turbopack may create multiple loader instances, but each
+// only handles file transformation. The ExtractionCompiler (which manages
+// catalogs) is initialized separately in createNextIntlPlugin.
+let extractor;
 function extractionLoader(source) {
-  const options = this.getOptions();
   const callback = this.async();
-  if (!compiler) {
-    compiler = new ExtractionCompiler(options, {
-      // Avoid rollup's `replace` plugin to compile this away
-      isDevelopment: process.env['NODE_ENV'.trim()] === 'development',
+  const projectRoot = this.rootContext;
+
+  // Avoid rollup's `replace` plugin to compile this away
+  const isDevelopment = process.env['NODE_ENV'.trim()] === 'development';
+  if (!extractor) {
+    extractor = new MessageExtractor({
+      isDevelopment,
+      projectRoot,
       sourceMap: this.sourceMap
     });
   }
-  compiler.compile(this.resourcePath, source).then(result => {
+  extractor.extract(this.resourcePath, source).then(result => {
     callback(null, result.code, result.map);
   }).catch(callback);
 }

package/dist/esm/development/extractor/extractor/MessageExtractor.js

@@ -1,18 +1,19 @@
 import { createRequire } from 'module';
 import path from 'path';
 import { transform } from '@swc/core';
+import { getDefaultProjectRoot } from '../utils.js';
 import LRUCache from './LRUCache.js';
 
 const require = createRequire(import.meta.url);
 class MessageExtractor {
   compileCache = (() => new LRUCache(750))();
   constructor(opts) {
-    this.isDevelopment = opts.isDevelopment;
-    this.projectRoot = opts.projectRoot;
+    this.isDevelopment = opts.isDevelopment ?? false;
+    this.projectRoot = opts.projectRoot ?? getDefaultProjectRoot();
     this.sourceMap = opts.sourceMap ?? false;
   }
-  async processFileContent(absoluteFilePath, source) {
-    const cacheKey = source;
+  async extract(absoluteFilePath, source) {
+    const cacheKey = [source, absoluteFilePath].join('!');
     const cached = this.compileCache.get(cacheKey);
     if (cached) return cached;
 

package/dist/esm/development/extractor/format/ExtractorCodec.js

@@ -0,0 +1,5 @@
+function defineCodec(factory) {
+  return factory;
+}
+
+export { defineCodec };

package/dist/esm/development/extractor/format/codecs/JSONCodec.js

@@ -0,0 +1,40 @@
+import { getSortedMessages, setNestedProperty } from '../../utils.js';
+import { defineCodec } from '../ExtractorCodec.js';
+
+var JSONCodec = defineCodec(() => ({
+  decode(source) {
+    const json = JSON.parse(source);
+    const messages = [];
+    traverseMessages(json, (message, id) => {
+      messages.push({
+        id,
+        message
+      });
+    });
+    return messages;
+  },
+  encode(messages) {
+    const root = {};
+    for (const message of getSortedMessages(messages)) {
+      setNestedProperty(root, message.id, message.message);
+    }
+    return JSON.stringify(root, null, 2) + '\n';
+  },
+  toJSONString(source) {
+    return source;
+  }
+}));
+function traverseMessages(obj, callback, path = '') {
+  const NAMESPACE_SEPARATOR = '.';
+  for (const key of Object.keys(obj)) {
+    const newPath = path ? path + NAMESPACE_SEPARATOR + key : key;
+    const value = obj[key];
+    if (typeof value === 'string') {
+      callback(value, newPath);
+    } else if (typeof value === 'object') {
+      traverseMessages(value, callback, newPath);
+    }
+  }
+}
+
+export { JSONCodec as default };