jekyll-l10n 2.0.0 → 3.0.0

data/lib/jekyll-l10n/translation/libre_translator.rb

@@ -3,21 +3,27 @@
 require 'json'
 require 'net/http'
 require 'uri'
+require_relative '../constants'
+require_relative '../instrumentation'
 require_relative '../utils/error_handler'
 require_relative '../utils/logger_formatter'
+require_relative 'libre_translate_languages'
+require_relative 'request_slots'
+require_relative 'translation_progress'
 
 module Jekyll
   module L10n
     # Translates compendium entries using the LibreTranslate API.
     #
     # LibreTranslator integrates with the LibreTranslate API to automatically
-    # translate untranslated entries in compendium PO files. It handles batching,
-    # retries with exponential backoff, progress logging, and API error handling.
-    # Translations are written directly to POEntry objects in-place.
+    # translate untranslated entries in compendium PO files. It sends several
+    # requests at once, retries with exponential backoff, logs progress, and
+    # handles API errors. Translations are written directly to POEntry objects
+    # in-place.
     #
     # Key responsibilities:
-    # * Send translation requests to LibreTranslate API
-    # * Batch translations for efficiency
+    # * Map the site locale to a language code the server lists
+    # * Send translation requests to LibreTranslate API, several at once
     # * Handle API timeouts and failures with retries
     # * Log translation progress at configurable intervals
     # * Update POEntry msgstr with translated text
@@ -39,33 +45,43 @@ module Jekyll
       #   - libretranslate_api_url [String] API endpoint URL
       #   - libretranslate_api_key [String, nil] Optional API key
       #   - libretranslate_timeout [Integer] Request timeout in seconds
-      #   - libretranslate_batch_size [Integer] Entries per batch request
+      #   - libretranslate_batch_size [Integer] Most entries a worker takes at a time
+      #   - libretranslate_concurrency [Integer] Requests sent at once
       #   - libretranslate_retry_attempts [Integer] Max retry attempts
       #   - libretranslate_retry_delay [Integer] Delay between retries in seconds
       #   - libretranslate_progress_interval [Integer] Log progress every N entries
+      #   - libretranslate_mark_fuzzy? [Boolean] Store translations as fuzzy entries
       def initialize(config)
         @config = config
+        @lock = Mutex.new
+        @stopped = false
       end
 
       # Translate a compendium to a target locale.
       #
       # Identifies untranslated entries (empty msgstr) and sends them to LibreTranslate
-      # API for translation. Updates POEntry objects in-place with translated text.
-      # Handles batching, retries, and progress logging.
+      # API for translation, libretranslate_concurrency requests at a time. Updates
+      # POEntry objects in-place with translated text. Skips the locale, with a
+      # warning, when the server doesn't list its language or the source language.
       #
       # @param po_entries [Array<GetText::POEntry>] Array of PO entries from compendium
-      # @param target_locale [String] Target locale code (e.g., 'es', 'fr')
+      # @param target_locale [String] Target locale code (e.g., 'es', 'pt_BR')
       # @return [void]
       # @raise [TranslationError] If API request fails and stop_on_error is true
       # @raise [TranslationError] If max retry attempts exceeded
       def translate_compendium(po_entries, target_locale)
-        translatable_count, empty_entries = count_translatable_entries(po_entries)
-        return if empty_entries.empty?
+        empty_entries = untranslated_entries(po_entries)
+        return if empty_entries.empty? || @stopped
 
-        log_translation_progress(empty_entries.length, translatable_count, target_locale)
-        start_time = Time.now
-        process_translation_batches(empty_entries, target_locale, start_time)
-        log_translation_complete(empty_entries.length, target_locale, start_time)
+        api_source = api_locale_for(@config.libretranslate_source_locale, target_locale, empty_entries.length)
+        api_target = api_source && api_locale_for(target_locale, target_locale, empty_entries.length)
+        return unless api_target
+
+        log_translation_start(empty_entries.length, target_locale)
+        progress = TranslationProgress.new(total: empty_entries.length, locale: target_locale,
+                                           interval: @config.libretranslate_progress_interval)
+        process_translation_batches(empty_entries, api_target, api_source, progress)
+        progress.log_summary
       end
 
       private
@@ -99,7 +115,7 @@ module Jekyll
 
           delay_str = delay.round(2)
           Jekyll.logger.warn(
-            'Localization',
+            Constants::LOG_PREFIX,
             "Retrying translation (attempt #{attempts}/#{max_attempts}) after #{delay_str}s: " \
             "#{e.message}"
           )
@@ -108,85 +124,122 @@ module Jekyll
         end
       end
 
-      def count_translatable_entries(po_entries)
-        translatable_count = po_entries.count { |e| !e.msgid.strip.empty? }
-        empty_count = po_entries.select { |entry| entry.msgstr.empty? && !entry.msgid.strip.empty? }
-        [translatable_count, empty_count]
+      def untranslated_entries(po_entries)
+        po_entries.select { |entry| entry.msgstr.empty? && !entry.msgid.strip.empty? }
       end
 
-      def log_translation_progress(empty_count, _translatable_count, target_locale)
-        Jekyll.logger.info 'Localization',
-                           "Translating #{empty_count} entries for #{target_locale}"
+      # The server's code for locale (the source or target locale), or nil to
+      # skip target_locale.
+      def api_locale_for(locale, target_locale, entry_count)
+        api_locale = languages.api_code(locale)
+        unless api_locale
+          Jekyll.logger.warn Constants::LOG_PREFIX,
+                             "[LibreTranslator] LibreTranslate does not support #{locale}; " \
+                             "skipping #{entry_count} entries for #{target_locale}"
+        end
+        api_locale
+      rescue LibreTranslateLanguages::Unavailable => e
+        raise TranslationError, e.message if @config.libretranslate_stop_on_error?
+
+        Jekyll.logger.warn Constants::LOG_PREFIX,
+                           "[LibreTranslator] #{e.message}; skipping #{entry_count} entries for #{target_locale}"
+        nil
       end
 
-      def log_periodic_progress(current_num, total_entries, target_locale, start_time)
-        elapsed = Time.now - start_time
-        percent = (current_num.to_f / total_entries * 100).round
+      def languages
+        @languages ||= LibreTranslateLanguages.new(@config)
+      end
 
-        Jekyll.logger.info 'Localization',
-                           "[LibreTranslator] Progress: #{current_num}/#{total_entries} " \
-                           "(#{percent}%) translated for #{target_locale} [#{elapsed.round(1)}s]"
+      def log_translation_start(entry_count, target_locale)
+        Jekyll.logger.info Constants::LOG_PREFIX,
+                           "Translating #{entry_count} entries for #{target_locale}"
       end
 
-      def log_translation_complete(entries_translated, target_locale, start_time)
-        elapsed = Time.now - start_time
-        throughput = entries_translated.positive? ? (entries_translated / elapsed).round(1) : 0
+      # Workers take batches from a shared queue until it is empty. A worker
+      # that fails stops this translator, so the workers of every locale stop
+      # after their current request, and its error is raised once this
+      # locale's workers have stopped. A batch
+      # holds at most an equal share of the entries, so a batch size larger
+      # than that share doesn't leave workers without one.
+      def process_translation_batches(empty_entries, api_locale, api_source, progress)
+        queue = Queue.new
+        share = (empty_entries.size.to_f / @config.libretranslate_concurrency).ceil
+        empty_entries.each_slice([@config.libretranslate_batch_size, share].min) { |batch| queue << batch }
+        queue.close
 
-        Jekyll.logger.info 'Localization',
-                           "[LibreTranslator] Completed: #{entries_translated} entries " \
-                           "translated for #{target_locale} [#{elapsed.round(1)}s, " \
-                           "~#{throughput} entries/sec]"
+        workers = Array.new([@config.libretranslate_concurrency, queue.size].min) do
+          Instrumentation.traced_thread { work_through(queue, api_locale, api_source, progress) }
+        end
+        join_workers(workers)
       end
 
-      def process_translation_batches(empty_entries, target_locale, start_time)
-        batch_size = @config.libretranslate_batch_size
-        progress_interval = @config.libretranslate_progress_interval
-
-        empty_entries.each_slice(batch_size).with_index do |batch, batch_num|
-          start_num = (batch_num * batch_size) + 1
-          retry_with_backoff do
-            translate_batch(batch, target_locale, start_num, empty_entries.length,
-                            start_time, progress_interval)
-          end
+      def work_through(queue, api_locale, api_source, progress)
+        Thread.current.report_on_exception = false
+        while !@stopped && (batch = queue.pop)
+          translate_batch(batch, api_locale, api_source, progress)
         end
+      rescue StandardError
+        @stopped = true
+        raise
       end
 
-      # rubocop:disable Metrics/ParameterLists
-      def translate_batch(batch, target_locale, start_num, total_entries, start_time,
-                          progress_interval)
-        # rubocop:enable Metrics/ParameterLists
-        batch.each_with_index do |entry, index|
-          current_num = start_num + index
+      def join_workers(workers)
+        first_error = nil
+        workers.each do |worker|
+          worker.join
+        rescue StandardError => e
+          first_error ||= e
+        end
+        raise first_error if first_error
+      end
 
-          # Log progress at intervals and at the end
-          if progress_interval.positive? &&
-             ((current_num % progress_interval).zero? || current_num == total_entries)
-            log_periodic_progress(current_num, total_entries, target_locale, start_time)
-          end
+      # A failed request is retried on its own, so a retry never resends
+      # entries that were already translated.
+      def translate_batch(batch, api_locale, api_source, progress)
+        batch.each do |entry|
+          break if @stopped
 
-          # Move per-entry logs to TRACE level for deep debugging
           LoggerFormatter.trace_if_enabled(
-            @config, 'LibreTranslator',
-            "Translating entry #{current_num}/#{total_entries}: #{entry.msgid[0..50]}"
+            @config, 'LibreTranslator', "Translating entry to #{api_locale}: #{entry.msgid[0..50]}"
           )
 
-          texts = [entry.msgid]
-          translations = make_api_request(texts, target_locale)
-          entry.msgstr = translations&.first || ''
+          translations = retry_with_backoff do
+            request_slots.hold(api_locale) { make_api_request([entry.msgid], api_locale, api_source) }
+          end
+          msgstr = translations&.first || ''
+          store_translation(entry, msgstr)
+          progress.record(!msgstr.empty?)
         end
       rescue StandardError => e
         ErrorHandler.log_error('translate_batch', e)
         raise
       end
 
+      # Locales are translated at once, each with its own workers, so a
+      # worker takes one of libretranslate_concurrency slots, shared by every
+      # locale, for each request. Slots are keyed by the server's language
+      # code, so locales that use the same language model share its turns.
+      def request_slots
+        @lock.synchronize { @request_slots ||= RequestSlots.new(@config.libretranslate_concurrency) }
+      end
+
+      # With libretranslate_mark_fuzzy, a machine translation is stored as a
+      # fuzzy entry, which pages don't use until a person reviews it. Its
+      # msgstr is no longer empty, so later builds don't send it again.
+      def store_translation(entry, msgstr)
+        entry.msgstr = msgstr
+        entry.flag = 'fuzzy' if !msgstr.empty? && @config.libretranslate_mark_fuzzy?
+      end
+
+      # A new connection per request: workers send at once, and one
+      # Net::HTTP object must not be shared between threads.
       def http_config
-        @http_config ||= begin
-          uri = URI(@config.libretranslate_api_url)
-          http = Net::HTTP.new(uri.host, uri.port)
-          http.read_timeout = @config.libretranslate_timeout
-          http.open_timeout = @config.libretranslate_timeout
-          http
-        end
+        uri = URI(@config.libretranslate_api_url)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == 'https'
+        http.read_timeout = @config.libretranslate_timeout
+        http.open_timeout = @config.libretranslate_timeout
+        http
       end
 
       def api_headers
@@ -198,12 +251,15 @@ module Jekyll
         headers
       end
 
-      def make_api_request(text, target_locale)
+      # target_locale and source_locale are codes the server lists. The
+      # target comes before the source so the l10n.libretranslate_batch span
+      # (see Instrumentation) records it as l10n.locale.
+      def make_api_request(text, target_locale, source_locale)
         uri = URI("#{@config.libretranslate_api_url}/translate")
-        Jekyll.logger.debug 'Localization', "[LibreTranslator] Requesting #{uri}"
-        request = build_request(uri, text, target_locale)
+        Jekyll.logger.debug Constants::LOG_PREFIX, "[LibreTranslator] Requesting #{uri}"
+        request = build_request(uri, text, target_locale, source_locale)
         response = http_config.request(request)
-        Jekyll.logger.debug 'Localization', "[LibreTranslator] Response code: #{response.code}"
+        Jekyll.logger.debug Constants::LOG_PREFIX, "[LibreTranslator] Response code: #{response.code}"
         handle_api_response(response)
       rescue Net::ReadTimeout, Net::OpenTimeout => e
         raise TranslationError, "API timeout: #{e.message}"
@@ -212,11 +268,11 @@ module Jekyll
         nil
       end
 
-      def build_request(uri, text, target_locale)
+      def build_request(uri, text, target_locale, source_locale)
         request = Net::HTTP::Post.new(uri, api_headers)
         request.body = {
           q: text,
-          source: @config.libretranslate_source_locale,
+          source: source_locale,
           target: target_locale,
           format: @config.libretranslate_format
         }.to_json
@@ -227,14 +283,14 @@ module Jekyll
         response_data = JSON.parse(response.body)
 
         if response_data['error']
-          Jekyll.logger.warn 'Localization',
+          Jekyll.logger.warn Constants::LOG_PREFIX,
                              "[LibreTranslator] API rejected request: #{response_data['error']}"
           return nil
         end
 
         translated = response_data['translatedText']
         translation_str = translated.is_a?(Array) ? translated.inspect : translated
-        Jekyll.logger.debug 'Localization', "[LibreTranslator] Parsed translation: #{translation_str}"
+        Jekyll.logger.debug Constants::LOG_PREFIX, "[LibreTranslator] Parsed translation: #{translation_str}"
         translated
       rescue JSON::ParserError, NoMethodError => e
         raise TranslationError, "Invalid API response: #{e.message}"

data/lib/jekyll-l10n/translation/request_slots.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Jekyll
+  module L10n
+    # Limits how many requests are in flight at once, keyed by language. A
+    # freed slot goes to the waiting key with the fewest slots, not to the
+    # caller that waited longest, so keys that wait at the same time share
+    # the slots instead of taking turns in bursts.
+    #
+    # @example
+    #   slots = RequestSlots.new(4)
+    #   slots.hold('es') { send_request }
+    class RequestSlots
+      # @param limit [Integer] Slots available at once
+      def initialize(limit)
+        @free = limit
+        @held = Hash.new(0)
+        @waiting = Hash.new(0)
+        @lock = Mutex.new
+        @changed = ConditionVariable.new
+      end
+
+      # Run the block while holding a slot for key.
+      #
+      # @param key [String] Language the request is for
+      # @return [Object] The value of the block
+      def hold(key)
+        acquire(key)
+        begin
+          yield
+        ensure
+          release(key)
+        end
+      end
+
+      # Wait for a slot for key and take it.
+      #
+      # @param key [String] Language the request is for
+      # @return [void]
+      def acquire(key)
+        @lock.synchronize do
+          @waiting[key] += 1
+          @changed.wait(@lock) until @free.positive? && fewest_held?(key)
+          @waiting.delete(key) if (@waiting[key] -= 1).zero?
+          @free -= 1
+          @held[key] += 1
+        end
+      end
+
+      # Give back a slot taken for key.
+      #
+      # @param key [String] Language the request was for
+      # @return [void]
+      def release(key)
+        @lock.synchronize do
+          @free += 1
+          @held[key] -= 1
+          @changed.broadcast
+        end
+      end
+
+      private
+
+      def fewest_held?(key)
+        @held[key] <= @waiting.keys.map { |waiting_key| @held[waiting_key] }.min
+      end
+    end
+  end
+end

data/lib/jekyll-l10n/translation/translation_progress.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require_relative '../constants'
+
+module Jekyll
+  module L10n
+    # Counts the entries machine-translated for one locale and logs progress
+    # and a summary. Worker threads share one instance, so counting is
+    # synchronized.
+    #
+    # @example
+    #   progress = TranslationProgress.new(total: 20, locale: 'es', interval: 10)
+    #   progress.record(true)   # the entry got a translation
+    #   progress.record(false)  # the server returned none
+    #   progress.log_summary
+    class TranslationProgress
+      # @param total [Integer] Entries to translate
+      # @param locale [String] Locale named in the log messages
+      # @param interval [Integer] Log progress every this many entries; 0 turns it off
+      def initialize(total:, locale:, interval:)
+        @total = total
+        @locale = locale
+        @interval = interval
+        @done = 0
+        @translated = 0
+        @start_time = Time.now
+        @lock = Mutex.new
+      end
+
+      # Count one finished entry, logging progress when it is due.
+      #
+      # @param translated [Boolean] Whether the entry got a translation
+      # @return [void]
+      def record(translated)
+        done = @lock.synchronize do
+          @translated += 1 if translated
+          @done += 1
+        end
+        log_progress(done) if progress_due?(done)
+      end
+
+      # Log how many entries got a translation, how many failed, and the
+      # throughput.
+      #
+      # @return [void]
+      def log_summary
+        elapsed = Time.now - @start_time
+        throughput = @done.positive? ? (@done / elapsed).round(1) : 0
+        failed = @done - @translated
+        failures = failed.positive? ? ", #{failed} failed" : ''
+
+        Jekyll.logger.info Constants::LOG_PREFIX,
+                           "[LibreTranslator] Completed: #{@translated}/#{@total} entries " \
+                           "translated for #{@locale}#{failures} [#{elapsed.round(1)}s, " \
+                           "~#{throughput} entries/sec]"
+      end
+
+      private
+
+      # Progress is logged every interval entries and after the last one.
+      def progress_due?(done)
+        return false unless @interval.positive?
+
+        (done % @interval).zero? || done == @total
+      end
+
+      def log_progress(done)
+        elapsed = Time.now - @start_time
+        percent = (done.to_f / @total * 100).round
+
+        Jekyll.logger.info Constants::LOG_PREFIX,
+                           "[LibreTranslator] Progress: #{done}/#{@total} " \
+                           "(#{percent}%) translated for #{@locale} [#{elapsed.round(1)}s]"
+      end
+    end
+  end
+end

data/lib/jekyll-l10n/translation/translator.rb

@@ -67,11 +67,7 @@ module Jekyll
       end
 
       def apply_translations_to_page(original_page, config, translations, locale, baseurl)
-        translator = HtmlTranslator.new(
-          config.fallback_mode,
-          config.translatable_attributes,
-          debug_logging: config.debug_logging?
-        )
+        translator = HtmlTranslator.from_config(config)
 
         translated_html = apply_translations(translator, translations, locale, baseurl)
         return unless translated_html
@@ -79,6 +75,7 @@ module Jekyll
         # Preserve external link icons from the original page after translation
         translated_html = preserve_external_link_icons(original_page.output, translated_html)
         @page.output = translated_html
+        @page.translations_digest = translations.hash if @page.respond_to?(:translations_digest=)
       end
 
       def preserve_external_link_icons(original_html, translated_html)
@@ -102,18 +99,24 @@ module Jekyll
       end
 
       def should_translate?
-        has_data = @page.data
-        has_locale = has_data && @page.data['locale']
-        has_original_url = has_data && @page.data['original_url']
-        has_output = @page.output
-        output_not_empty = !@page.output&.empty?
-
-        msg = "should_translate? data=#{has_data}, locale=#{has_locale}, " \
-              "original_url=#{has_original_url}, output=#{has_output}, " \
-              "not_empty=#{output_not_empty}"
-        LoggerFormatter.debug_if_enabled('Translator', msg)
-
-        has_data && has_locale && has_original_url && has_output && output_not_empty
+        checks = translation_checks
+        details = checks.map { |name, value| "#{name}=#{value}" }.join(', ')
+        LoggerFormatter.debug_if_enabled('Translator', "should_translate? #{details}")
+
+        checks.values.all?
+      end
+
+      # Each value must be truthy for the page to be translated.
+      def translation_checks
+        data = @page.data
+        output = @page.output
+        {
+          data: data,
+          locale: data && data['locale'],
+          original_url: data && data['original_url'],
+          output: output,
+          not_empty: !output&.empty?
+        }
       end
 
       def find_original_page_by_url(url)

data/lib/jekyll-l10n/utils/baseurl_normalizer.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require 'set'
+
+module Jekyll
+  module L10n
+    # Keeps the site baseurl out of extracted strings.
+    #
+    # A string with markup, such as a logo <img> in a link, carries its URLs
+    # into the msgid. Those URLs include the site baseurl, which changes between
+    # builds (local, branch previews, production). Without this step the same
+    # string gets a new msgid for every baseurl, which retires the old entry,
+    # adds a fuzzy one, and changes every compendium on each such build.
+    #
+    # Extraction strips the baseurl from URL attributes before reading strings.
+    # Translation strips it the same way, so lookups match those msgids, and
+    # restores it after translations are applied, including in markup that
+    # came from a msgstr.
+    #
+    # @example
+    #   state = BaseurlNormalizer.strip(doc, '/docs')
+    #   # <img src="/docs/logo.svg"> now reads <img src="/logo.svg">
+    #   BaseurlNormalizer.restore(doc, state)
+    #   # and reads <img src="/docs/logo.svg"> again
+    module BaseurlNormalizer
+      # Attributes whose value is a single URL.
+      URL_ATTRIBUTES = %w[href src poster].freeze
+
+      URL_SELECTOR = URL_ATTRIBUTES.map { |attr| "[#{attr}]" }.join(',').freeze
+      private_constant :URL_SELECTOR
+
+      # What strip changed, for restore.
+      #
+      # originals: element => { attribute => value before strip }
+      # stripped: value after strip => value before strip
+      # unchanged: root-relative values strip did not touch
+      State = Struct.new(:originals, :stripped, :unchanged)
+
+      module_function
+
+      # Remove the baseurl from URL attributes that start with it.
+      #
+      # Only whole path segments match: with baseurl "/docs", "/docs" and
+      # "/docs/page/" are stripped, "/docsite/" is not.
+      #
+      # @param doc [Nokogiri::XML::Node] Document or element (modified in place)
+      # @param baseurl [String, nil] Site baseurl, e.g. "/docs"
+      # @return [State, nil] What changed, or nil when there is no baseurl
+      def strip(doc, baseurl)
+        return nil if baseurl.nil? || baseurl.empty? || baseurl == '/'
+
+        state = State.new({}.compare_by_identity, {}, Set.new)
+        each_url_attribute(doc) do |element, attr, value|
+          path = path_below(value, baseurl)
+          if path
+            strip_attribute(state, element, attr, path)
+          elsif value.start_with?('/')
+            state.unchanged << value
+          end
+        end
+        state
+      end
+
+      # Put the baseurl back.
+      #
+      # Elements strip changed get their own values back. Other elements, such
+      # as markup injected from a msgstr, get the baseurl back on values strip
+      # produced, unless the page also had that same value without a baseurl.
+      #
+      # @param doc [Nokogiri::XML::Node] Document or element (modified in place)
+      # @param state [State, nil] The result of {.strip}
+      # @return [void]
+      def restore(doc, state)
+        return unless state
+
+        each_url_attribute(doc) do |element, attr, value|
+          original = state.originals.dig(element, attr)
+          original ||= state.stripped[value] unless state.unchanged.include?(value)
+          element[attr] = original if original
+        end
+      end
+
+      def strip_attribute(state, element, attr, path)
+        value = element[attr]
+        (state.originals[element] ||= {})[attr] = value
+        state.stripped[path] = value
+        element[attr] = path
+      end
+
+      def path_below(value, baseurl)
+        return '/' if value == baseurl
+
+        value[baseurl.length..] if value.start_with?("#{baseurl}/")
+      end
+
+      def each_url_attribute(doc)
+        doc.css(URL_SELECTOR).each do |element|
+          URL_ATTRIBUTES.each do |attr|
+            value = element[attr]
+            yield element, attr, value if value
+          end
+        end
+      end
+
+      private_class_method :strip_attribute, :path_below, :each_url_attribute
+    end
+  end
+end