jekyll-l10n 2.0.0 → 3.0.0

data/lib/jekyll-l10n/po_file/writer.rb

@@ -75,17 +75,23 @@ module Jekyll
 
           po_file[entry[:msgid]] = po_entry
         rescue StandardError => e
-          truncate_length = Jekyll::L10n::Constants::LOG_TRUNCATE_LONG
-          errors << "Error adding '#{entry[:msgid][0..truncate_length]}': " \
-                    "#{e.class} - #{e.message}"
+          errors << entry_error(entry, e)
         end
+        warn_merge_errors(errors)
+      end
 
-        return unless errors.any?
+      def self.warn_merge_errors(errors)
+        return if errors.empty?
 
-        Jekyll.logger.warn 'Localization',
+        Jekyll.logger.warn Constants::LOG_PREFIX,
                            "#{errors.length} errors during merging: #{errors.join(', ')}"
       end
 
+      def self.entry_error(entry, error)
+        truncate_length = Jekyll::L10n::Constants::LOG_TRUNCATE_LONG
+        "Error adding '#{entry[:msgid][0..truncate_length]}': #{error.class} - #{error.message}"
+      end
+
       def self.build_fuzzy_candidates(existing_po, new_msgids)
         existing_po.reject do |old_msgid, entry|
           old_msgid.empty? ||
@@ -113,23 +119,32 @@ module Jekyll
       def self.set_po_entry_msgstr(po_entry, existing_entry, new_msgstr,
                                    fuzzy_candidates = {})
         if existing_entry.is_a?(Hash)
-          po_entry.msgstr = existing_entry[:msgstr] || ''
-          po_entry.flag   = 'fuzzy' if existing_entry[:fuzzy]
-          po_entry.previous = existing_entry[:previous_msgid] if existing_entry[:previous_msgid]
+          apply_existing_entry(po_entry, existing_entry)
         elsif existing_entry
           po_entry.msgstr = existing_entry
         else
-          match = PoFuzzyMatcher.find_match(po_entry.msgid, fuzzy_candidates)
-          if match
-            po_entry.msgstr   = match[:msgstr]
-            po_entry.flag     = 'fuzzy'
-            po_entry.previous = match[:msgid]
-          else
-            po_entry.msgstr = new_msgstr || ''
-          end
+          apply_fuzzy_match_or_new(po_entry, new_msgstr, fuzzy_candidates)
         end
       end
 
+      def self.apply_existing_entry(po_entry, existing_entry)
+        po_entry.msgstr = existing_entry[:msgstr] || ''
+        po_entry.flag   = 'fuzzy' if existing_entry[:fuzzy]
+        po_entry.previous = existing_entry[:previous_msgid] if existing_entry[:previous_msgid]
+      end
+
+      def self.apply_fuzzy_match_or_new(po_entry, new_msgstr, fuzzy_candidates)
+        match = PoFuzzyMatcher.find_match(po_entry.msgid, fuzzy_candidates)
+        unless match
+          po_entry.msgstr = new_msgstr || ''
+          return
+        end
+
+        po_entry.msgstr   = match[:msgstr]
+        po_entry.flag     = 'fuzzy'
+        po_entry.previous = match[:msgid]
+      end
+
       def self.header(locale)
         header_str = "Language: #{locale}\nMIME-Version: 1.0\nContent-Type: text/plain; "
         "#{header_str}charset=UTF-8\nContent-Transfer-Encoding: 8bit\n"

data/lib/jekyll-l10n/translation/block_text_extractor.rb

@@ -2,7 +2,6 @@
 
 require_relative '../utils/html_text_utils'
 require_relative '../utils/html_elements'
-require_relative '../utils/text_validator'
 
 module Jekyll
   module L10n
@@ -27,19 +26,16 @@ module Jekyll
 
       # Extract normalized block text from an element.
       #
-      # Returns nil if element is not extractable or if extracted text fails
-      # validation. Clones the node, removes nested block elements and empty
-      # icon tags, normalizes whitespace, and validates. HTML entities are
+      # Returns the msgid extraction gives the element (HtmlTextUtils.block_text),
+      # or nil if it is not a content element or has no msgid. HTML entities are
       # preserved verbatim to match the keys produced by the extraction pipeline.
       #
       # @param node [Nokogiri::XML::Element] DOM element to extract from
       # @return [String, nil] Normalized text from element, or nil if not valid
       def extract(node)
         return nil unless extractable?(node)
-        return nil if HtmlTextUtils.layout_block_children?(node)
 
-        text = HtmlTextUtils.extract_with_inline_tags(node)
-        TextValidator.valid?(text) ? text : nil
+        HtmlTextUtils.block_text(node)
       end
 
       def extractable?(node)

data/lib/jekyll-l10n/translation/html_translator.rb

@@ -9,6 +9,7 @@ require_relative '../utils/html_elements'
 require_relative '../utils/html_parser'
 require_relative '../utils/text_validator'
 require_relative '../utils/html_text_utils'
+require_relative '../utils/baseurl_normalizer'
 require_relative 'block_text_extractor'
 
 module Jekyll
@@ -22,13 +23,13 @@ module Jekyll
     # translations for elements with complete translations and preserves URL transformations.
     #
     # Key responsibilities:
-    # * Parse full HTML documents while preserving DOCTYPE and structure
+    # * Parse full HTML documents (through HtmlParser)
     # * Translate text nodes using normalized text for matching
     # * Translate HTML attributes (title, alt, aria-label, placeholder, aria-description)
     # * Apply fallback modes when translations are missing (english/marker/empty)
     # * Handle block-level translations for content elements
+    # * Look strings up without the site baseurl, as extraction stores them
     # * Transform relative URLs to locale-prefixed URLs
-    # * Remove auto-inserted meta charset tags from serialized HTML
     #
     # @example
     #   translator = HtmlTranslator.new('english', ['title', 'alt'])
@@ -55,6 +56,14 @@ module Jekyll
         @debug_logging = debug_logging
       end
 
+      # Build a translator from a page's localization settings.
+      #
+      # @param config [PageLocalesConfig] The page's settings
+      # @return [HtmlTranslator]
+      def self.from_config(config)
+        new(config.fallback_mode, config.translatable_attributes, debug_logging: config.debug_logging?)
+      end
+
       # Translate an HTML document to a specific locale.
       #
       # Parses the HTML document, applies translations to text nodes and attributes,
@@ -66,36 +75,28 @@ module Jekyll
       #   or metadata hashes with :msgstr, :reference, :fuzzy keys
       # @param locale [String] Target locale code (defaults to "en"; e.g., 'es', 'fr')
       # @param baseurl [String] Base URL for relative URL transformation (defaults to "")
-      # @return [String] Translated HTML with URLs transformed and meta charset removed
+      # @return [String] Translated HTML with URLs transformed
       def translate(html, translations, locale = 'en', baseurl = '')
-        # Use HtmlParser to properly parse full HTML documents while preserving
-        # DOCTYPE, html tag, and document structure. Any auto-inserted meta tags are
-        # removed by HtmlParser.remove_meta_charset after serialization.
-        # See: spec/regression/nokogiri_meta_tag_spec.rb for regression tests
         doc = HtmlParser.parse_document(html)
 
+        # Block text of each content element, keyed by node identity, so a
+        # paragraph with several text nodes is extracted once instead of up to
+        # twice per text node. Cleared in the ensure block below.
+        @block_texts = {}.compare_by_identity
+
+        # Look strings up without the baseurl, as extraction stored them.
+        baseurl_state = BaseurlNormalizer.strip(doc, baseurl)
         translate_node(doc, translations)
+        BaseurlNormalizer.restore(doc, baseurl_state)
 
-        # Transform URLs on the document object before serialization to avoid double-parsing
-        # and preserve the correct DOCTYPE and HTML structure. This prevents Nokogiri from
-        # downgrading to HTML 4.0 DOCTYPE when parsing the serialized HTML again.
-        # See: spec/jekyll-l10n/utils/url_transformer_spec.rb for tests
+        # Transform URLs on the parsed document, so the page is parsed once.
         UrlTransformer.transform_document(doc, locale, baseurl)
 
-        result = doc.to_html
-
-        # Remove the auto-inserted meta tag by libxml2 during HTML serialization
-        # Matches: <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
-        HtmlParser.remove_meta_charset(result)
+        doc.to_html
+      ensure
+        @block_texts = nil
       end
 
-      # Inline element tags that are indexed and restored during block-level translation.
-      # Includes translatable-content tags (replaced with <g> in Phase 2) and
-      # literal-content tags (class/style stripped but tag preserved). Derived from the
-      # canonical taxonomy in HtmlTextUtils so extraction and injection never diverge.
-      INLINE_ELEMENT_TAGS = (HtmlTextUtils::PLACEHOLDERED_INLINE_TAGS +
-                             HtmlTextUtils::LITERAL_INLINE_TAGS).freeze
-
       private
 
       def translate_node(node, translations)
@@ -114,7 +115,8 @@ module Jekyll
         log_text_node_debug(text) if should_log_text_debug?(text)
 
         normalized_text = TextNormalizer.normalize(text)
-        translated = TranslationResolver.resolve(node, normalized_text, translations)
+        translated = TranslationResolver.resolve(node, normalized_text, translations,
+                                                 block_texts: @block_texts)
 
         return if apply_block_level_translation?(node, normalized_text, translations)
 
@@ -126,113 +128,149 @@ module Jekyll
         node.content = apply_fallback(original_content, translated)
       end
 
-      def translate_node_attributes(node, translations)
-        @translatable_attrs.each do |attr_name|
-          value = node[attr_name]
-          next if value.nil? || value.empty?
+      def should_skip_translation?(text)
+        !TextValidator.valid?(text)
+      end
 
-          value = value.strip
-          next unless TextValidator.valid?(value)
+      def should_log_text_debug?(text)
+        @debug_logging && text.include?('attribute')
+      end
 
-          normalized_value = TextNormalizer.normalize(value)
-          translated = translations[normalized_value]
-          node[attr_name] = apply_fallback(value, translated)
+      def log_text_node_debug(text)
+        truncate_length = Jekyll::L10n::Constants::LOG_TRUNCATE_MEDIUM
+        message = '[HtmlTranslator] Processing text node: ' \
+                  "#{text[0..truncate_length]}... (#{text.length} chars)"
+        Jekyll.logger.info Constants::LOG_PREFIX, message
+      end
+
+      def apply_block_level_translation?(node, normalized_text, translations)
+        ancestor = TranslationResolver.find_content_element_ancestor(node)
+        return false unless ancestor
+
+        return false if TranslationResolver.contains_protected_elements?(ancestor)
+
+        block_text = TranslationResolver.block_text_for(ancestor, @block_texts)
+        return false unless block_text && block_text != normalized_text
+
+        translation = select_block_translation(node, ancestor, normalized_text, block_text,
+                                               translations)
+        return false unless translation
+
+        apply_block_level_translation(ancestor, translation, translations)
+        true
+      end
+
+      # When text is directly inside a content element, prefer its direct translation
+      # (the msgstr may itself contain HTML, e.g. a heading with an anchor).
+      # When text is nested inside inline element(s), only apply if the full block
+      # text is an explicit translation key.
+      def select_block_translation(node, ancestor, normalized_text, block_text, translations)
+        if node.parent == ancestor
+          translations[normalized_text] || translations[block_text]
+        else
+          translations[block_text]
         end
       end
 
-      def apply_block_level_translation(parent, translation)
+      def apply_block_level_translation(parent, translation, translations)
         if @debug_logging
-          Jekyll.logger.info 'Localization',
+          Jekyll.logger.info Constants::LOG_PREFIX,
                              "[HtmlTranslator] BLOCK-LEVEL TRANSLATION for <#{parent.name}>"
           truncate_length = Jekyll::L10n::Constants::LOG_TRUNCATE_LONG
-          Jekyll.logger.info 'Localization',
+          Jekyll.logger.info Constants::LOG_PREFIX,
                              "[HtmlTranslator]   Translation: #{translation[0..truncate_length]}..."
         end
 
-        inject_translation_preserving_structure(parent, translation)
+        inject_translation_preserving_structure(parent, translation, translations)
       end
 
-      def inject_translation_preserving_structure(parent, translation)
-        original_inline   = index_original_inline_elements(parent)
+      def inject_translation_preserving_structure(parent, translation, translations)
+        markup            = HtmlTextUtils.inline_markup(parent)
         preserved_blocks  = parent.css('pre').map(&:dup)
         preserved_anchors = parent.css('.heading-anchor').map(&:dup)
-        parent.children.each(&:remove)
-        parent.inner_html = translation
-        restore_inline_elements(parent, original_inline)
+        parent.children = HtmlParser.parse_fragment(translation, context: parent)
+        restore_markup(parent, markup, translations)
         preserved_blocks.each  { |b| parent.add_child(b) }
         preserved_anchors.each { |a| parent.add_child(a) } if parent.css('.heading-anchor').empty?
+        # Evict the rewritten parent from the block text cache: its text content
+        # has changed and any cached value is now stale.
+        @block_texts.delete(parent)
       end
 
-      def restore_inline_elements(parent, original_inline)
-        if parent.css('g[id]').any?
-          expand_g_placeholders(parent, original_inline)
-        else
-          restore_structural_attributes(parent, original_inline)
-        end
+      # Put back the markup inline_markup took out of the block's text.
+      def restore_markup(parent, markup, translations)
+        restored = expand_placeholders(parent, markup.placeholders) +
+                   restore_literal_attributes(parent, markup.literals)
+        restored.each { |element| translate_restored_attributes(element, translations) }
       end
 
-      def walk_inline_elements(node, &block)
-        node.children.each do |child|
-          next unless child.element?
-
-          if INLINE_ELEMENT_TAGS.include?(child.name)
-            yield child
-          else
-            walk_inline_elements(child, &block)
+      # Turn each <g id="N"> and <x id="N"> back into the element it stands for,
+      # with that element's tag and attributes, innermost first so an outer
+      # placeholder moves already-restored children. A placeholder whose id has
+      # no element is unwrapped to its children, so stray tags never render.
+      #
+      # @return [Array<Nokogiri::XML::Element>] The restored elements
+      def expand_placeholders(parent, placeholders)
+        parent.css('g[id], x[id]').reverse.filter_map do |placeholder|
+          tag, attrs = placeholders[placeholder['id']]
+          unless tag
+            placeholder.replace(placeholder.children)
+            next
           end
+
+          element = placeholder.document.create_element(tag)
+          attrs.each { |name, value| element[name] = value }
+          element.children = placeholder.children
+          placeholder.replace(element)
+          element
         end
       end
 
-      def index_original_inline_elements(parent)
-        index = {}
-        n = 1
-        walk_inline_elements(parent) do |el|
-          index[n] = { tag: el.name, attrs: el.attributes.transform_values(&:value) }
-          n += 1
+      # Restored elements carry the page's attributes, and the walk never visits
+      # them. Translate the ones that have a translation; the others may already
+      # have been translated before the block was rewritten.
+      def translate_restored_attributes(element, translations)
+        @translatable_attrs.each do |attr_name|
+          value = element[attr_name]&.strip
+          next unless value && TextValidator.valid?(value)
+
+          translated = translations[TextNormalizer.normalize(value)]
+          element[attr_name] = translated if translated && !translated.empty?
         end
-        index
       end
 
-      # Phase 2 path: expand <g id="N"> placeholders to original elements with full
-      # attributes. The id corresponds to the Nth *placeholdered* inline element in the
-      # source DOM. Literal-content elements (<code>, <var>, <kbd>, <samp>) are never
-      # placeholdered and are excluded from the numbering, matching the extraction side
-      # in HtmlTextUtils.top_level_placeholdered_inline_elements. A <g> whose id has no
-      # source counterpart is unwrapped to its children so stray <g> tags never render.
-      def expand_g_placeholders(parent, original_inline)
-        placeholdered = original_inline.values.select do |orig|
-          HtmlTextUtils::PLACEHOLDERED_INLINE_TAGS.include?(orig[:tag])
-        end
-        parent.css('g[id]').each do |g|
-          orig = placeholdered[g['id'].to_i - 1]
-          g.replace(orig ? build_inline_element(g, orig) : g.children)
+      # Give literal elements (<code> etc.) their attributes back by position.
+      #
+      # @return [Array<Nokogiri::XML::Element>] The elements given attributes
+      def restore_literal_attributes(parent, literals)
+        literal_elements = parent.css(HtmlTextUtils::LITERAL_INLINE_TAGS.join(','))
+        literal_elements.zip(literals).filter_map do |element, (tag, attrs)|
+          next unless tag == element.name
+
+          attrs.each { |name, value| element[name] = value }
+          element
         end
       end
 
-      # Build a replacement element for a <g> placeholder from a source-DOM spec,
-      # copying the original tag name and all attributes and moving the <g> children in.
-      def build_inline_element(placeholder, orig)
-        node = placeholder.document.create_element(orig[:tag])
-        orig[:attrs].each { |name, val| node[name] = val }
-        placeholder.children.each { |child| node.add_child(child.dup) }
-        node
+      def log_translation_debug_info(text, normalized_text, translated, translations)
+        translation_data = DebugLogger::TranslationData.new(text: text,
+                                                            normalized_text: normalized_text,
+                                                            translated: translated,
+                                                            translations: translations)
+        DebugLogger.log_translation_details(self, translation_data)
       end
 
-      # Phase 1 path: restore structural attributes on injected elements by position.
-      # Matches each inline element in the translated content to its original counterpart
-      # by sequential order and overwrites structural attrs (class, style, id, etc.)
-      # with the values from the source DOM.
-      def restore_structural_attributes(parent, original_inline)
-        structural = Constants::STRUCTURAL_PASSTHROUGH_ATTRS
-        n = 1
-        walk_inline_elements(parent) do |el|
-          orig = original_inline[n]
-          n += 1
-          next unless orig && el.name == orig[:tag]
-
-          orig[:attrs].each do |name, val|
-            el[name] = val if structural.include?(name)
-          end
+      def translate_node_attributes(node, translations)
+        @translatable_attrs.each do |attr_name|
+          value = node[attr_name]
+          next if value.nil? || value.empty?
+
+          value = value.strip
+          next unless TextValidator.valid?(value)
+
+          normalized_value = TextNormalizer.normalize(value)
+          translated = translations[normalized_value]
+          node[attr_name] = apply_fallback(value, translated)
         end
       end
 
@@ -271,65 +309,6 @@ module Jekyll
           msgid
         end
       end
-
-      def log_text_node_debug(text)
-        truncate_length = Jekyll::L10n::Constants::LOG_TRUNCATE_MEDIUM
-        message = '[HtmlTranslator] Processing text node: ' \
-                  "#{text[0..truncate_length]}... (#{text.length} chars)"
-        Jekyll.logger.info 'Localization', message
-      end
-
-      def should_skip_translation?(text)
-        !TextValidator.valid?(text)
-      end
-
-      def should_log_text_debug?(text)
-        @debug_logging && text.include?('attribute')
-      end
-
-      def apply_block_level_translation?(node, normalized_text, translations)
-        ancestor = TranslationResolver.find_content_element_ancestor(node)
-        return false unless ancestor
-
-        return false if TranslationResolver.contains_protected_elements?(ancestor)
-
-        block_text = BlockTextExtractor.extract(ancestor)
-        return false unless block_text && block_text != normalized_text
-
-        translation = select_block_translation(node, ancestor, normalized_text, block_text,
-                                               translations)
-        return false unless translation
-
-        apply_block_level_translation(ancestor, translation)
-        true
-      end
-
-      # When text is directly inside a content element, prefer its direct translation
-      # (the msgstr may itself contain HTML, e.g. a heading with an anchor).
-      # When text is nested inside inline element(s), only apply if the full block
-      # text is an explicit translation key.
-      def select_block_translation(node, ancestor, normalized_text, block_text, translations)
-        if node.parent == ancestor
-          translations[normalized_text] || translations[block_text]
-        else
-          translations[block_text]
-        end
-      end
-
-      def log_translation_debug_info(text, normalized_text, translated, translations)
-        translation_data = DebugLogger::TranslationData.new(text: text,
-                                                            normalized_text: normalized_text,
-                                                            translated: translated,
-                                                            translations: translations)
-        DebugLogger.log_translation_details(self, translation_data)
-      end
-
-      private :log_text_node_debug, :should_skip_translation?, :should_log_text_debug?,
-              :apply_block_level_translation?, :select_block_translation,
-              :log_translation_debug_info,
-              :inject_translation_preserving_structure, :restore_inline_elements,
-              :walk_inline_elements, :index_original_inline_elements,
-              :expand_g_placeholders, :build_inline_element, :restore_structural_attributes
     end
   end
 end

data/lib/jekyll-l10n/translation/libre_translate_languages.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+require 'uri'
+
+module Jekyll
+  module L10n
+    # Maps site locales to the language codes a LibreTranslate server lists
+    # at /languages. Sites use codes like es_ES and pt_BR; the server uses
+    # es and pt-BR, and rejects codes it doesn't list.
+    #
+    # @example
+    #   languages = LibreTranslateLanguages.new(config)
+    #   languages.api_code('pt_BR') # => "pt-BR"
+    #   languages.api_code('es_ES') # => "es"
+    #   languages.api_code('xx')    # => nil
+    class LibreTranslateLanguages
+      # The language list could not be read from the server.
+      class Unavailable < StandardError; end
+
+      # @param config [PageLocalesConfig] Provides libretranslate_api_url and
+      #   libretranslate_timeout
+      def initialize(config)
+        @config = config
+        @lock = Mutex.new
+      end
+
+      # The server's code for a locale: the same language and region if the
+      # server lists it, else the base language.
+      #
+      # @param locale [String] Site locale, such as es_ES
+      # @return [String, nil] The server's code, or nil if it lists neither
+      # @raise [Unavailable] If the language list can't be read
+      def api_code(locale)
+        wanted = locale.tr('_', '-')
+        base = wanted.split('-').first
+        codes.find { |code| code.casecmp?(wanted) } || codes.find { |code| code.casecmp?(base) }
+      end
+
+      private
+
+      # Read once and kept; a failed read is tried again on the next call.
+      def codes
+        @lock.synchronize { @codes ||= fetch_codes }
+      end
+
+      def fetch_codes
+        response = request_languages
+        raise Unavailable, "#{languages_uri} answered HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess)
+
+        JSON.parse(response.body).map { |language| language.fetch('code') }
+      rescue JSON::ParserError, TypeError, KeyError, NoMethodError
+        raise Unavailable, "#{languages_uri} did not return a language list"
+      end
+
+      def request_languages
+        uri = languages_uri
+        timeout = @config.libretranslate_timeout
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https',
+                                            open_timeout: timeout, read_timeout: timeout) do |http|
+          http.get(uri.request_uri)
+        end
+      rescue SystemCallError, SocketError, Net::OpenTimeout, Net::ReadTimeout => e
+        raise Unavailable, "could not read #{uri}: #{e.message}"
+      end
+
+      def languages_uri
+        URI("#{@config.libretranslate_api_url}/languages")
+      end
+    end
+  end
+end