jekyll-l10n 1.0.5

data/lib/jekyll-l10n/po_file/writer.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+require "gettext/po"
+require "fileutils"
+require_relative "reader"
+require_relative "../constants"
+require_relative "../utils/file_operations"
+
+module Jekyll
+  module L10n
+    # Writes extraction entries to GNU Gettext PO files.
+    #
+    # PoFileWriter serializes extraction entries into standard PO file format with
+    # proper escaping, metadata, and optional merging of existing translations. It
+    # handles multi-line strings, special characters, fuzzy flags, and reference
+    # comments. When merging is enabled, existing translations are preserved while
+    # new strings are added or marked as fuzzy.
+    #
+    # Key responsibilities:
+    # * Create PO file entries from extraction data
+    # * Merge new entries with existing translations
+    # * Escape special characters and line breaks
+    # * Format multi-line and long strings properly
+    # * Add reference comments (file location references)
+    # * Add fuzzy flags for merge operations
+    # * Setup PO file headers with encoding information
+    # * Write UTF-8 encoded output
+    #
+    # @example
+    #   entries = [{ msgid: "Hello", msgstr: "", reference: "html/body/p[1]" }]
+    #   PoFileWriter.write('_locales/es.po', entries, 'es')
+    #   # Writes PO file with proper header and formatted entries
+    class PoFileWriter
+      # Write entries to a PO file.
+      #
+      # Creates or updates a PO file with the provided entries. If file exists and
+      # skip_merge is false, existing translations are merged (preserved while new
+      # entries added). Sets proper PO header with language and UTF-8 encoding.
+      #
+      # @param po_path [String] Full path to output PO file
+      # @param entries [Array<Hash>] Array of extraction entries, each with:
+      #   - :msgid [String] String to translate
+      #   - :msgstr [String] Translated string (typically empty for new extractions)
+      #   - :reference [String] File location reference for debugging
+      # @param locale [String] Locale code (e.g., 'es', 'fr')
+      # @param skip_merge [Boolean] If true, overwrite without merging existing translations
+      # @return [Boolean] True if successful
+      def self.write(po_path, entries, locale, skip_merge: false)
+        existing_po = if skip_merge || !File.exist?(po_path)
+                        {}
+                      else
+                        PoFileReader.parse_for_merge(po_path)
+                      end
+
+        po_file = ::GetText::PO.new
+        setup_po_header(po_file, locale)
+        merge_entries_preserving_translations(po_file, entries, existing_po)
+
+        output = serialize_po_file(po_file)
+        FileOperations.write_utf8(po_path, output)
+
+        true
+      end
+
+      def self.merge_entries_preserving_translations(po_file, entries, existing_po)
+        errors = []
+
+        entries.each do |entry|
+          po_entry = create_po_entry(entry, existing_po)
+          po_entry.add_comment(entry[:reference]) if entry[:reference]
+
+          po_file[entry[:msgid]] = po_entry
+        rescue StandardError => e
+          truncate_length = Jekyll::L10n::Constants::LOG_TRUNCATE_LONG
+          errors << "Error adding '#{entry[:msgid][0..truncate_length]}': " \
+                    "#{e.class} - #{e.message}"
+        end
+
+        return unless errors.any?
+
+        Jekyll.logger.warn "Localization",
+                           "#{errors.length} errors during merging: #{errors.join(", ")}"
+      end
+
+      def self.create_po_entry(entry, existing_po)
+        po_entry = ::GetText::POEntry.new(:normal)
+        po_entry.msgid = entry[:msgid]
+
+        existing_entry = existing_po[entry[:msgid]]
+        set_po_entry_msgstr(po_entry, existing_entry, entry[:msgstr])
+
+        po_entry
+      end
+
+      def self.set_po_entry_msgstr(po_entry, existing_entry, new_msgstr)
+        if existing_entry.is_a?(Hash)
+          po_entry.msgstr = existing_entry[:msgstr] || ""
+          po_entry.flag = "fuzzy" if existing_entry[:fuzzy]
+        else
+          po_entry.msgstr = existing_entry || (new_msgstr || "")
+        end
+      end
+
+      def self.header(locale)
+        header_str = "Language: #{locale}\nMIME-Version: 1.0\nContent-Type: text/plain; "
+        "#{header_str}charset=UTF-8\nContent-Transfer-Encoding: 8bit\n"
+      end
+
+      def self.setup_po_header(po_file, locale)
+        header_entry = ::GetText::POEntry.new(:normal)
+        header_entry.msgid = ""
+        header_entry.msgstr = header(locale)
+        po_file[""] = header_entry
+      end
+
+      def self.serialize_po_file(po_file)
+        lines = []
+        entries_list = get_entries_list(po_file)
+
+        entries_list.each do |entry|
+          serialize_po_entry(entry, lines)
+        end
+
+        lines.join("\n")
+      rescue StandardError => e
+        error_msg = "DEBUG serialize_po_file: Error iterating entries: #{e.class} - #{e.message}"
+        Jekyll.logger.info "Localization", error_msg
+        ""
+      end
+
+      def self.serialize_po_entry(entry, lines)
+        add_translator_comments(entry, lines)
+        add_flag_comment(entry, lines)
+        add_reference_comment(entry, lines)
+        add_msgid_msgstr(entry, lines)
+        lines << ""
+      end
+
+      def self.add_translator_comments(entry, lines)
+        return unless entry.translator_comment && !entry.translator_comment.empty?
+
+        entry.translator_comment.split("\n").each do |comment_line|
+          lines << "#  #{comment_line}" unless comment_line.empty?
+        end
+      end
+
+      def self.add_flag_comment(entry, lines)
+        flags = entry.flag.to_s.strip
+        lines << "#, #{flags}" unless flags.empty?
+      end
+
+      def self.add_reference_comment(entry, lines)
+        return unless entry.extracted_comment && !entry.extracted_comment.empty?
+
+        lines << "#: #{entry.extracted_comment}"
+      end
+
+      def self.add_msgid_msgstr(entry, lines)
+        lines << escape_po_string("msgid", entry.msgid)
+        lines << escape_po_string("msgstr", entry.msgstr.to_s)
+      end
+
+      def self.get_entries_list(po_file)
+        begin
+          po_file.entries
+        rescue StandardError
+          # entries method failed - try values as fallback
+          po_file.values
+        end
+      rescue StandardError => e
+        # Both methods failed - log and return empty
+        Jekyll.logger.warn "Localization", "Could not retrieve PO entries: #{e.class}"
+        []
+      end
+
+      def self.escape_po_string(prefix, value)
+        value = value.strip
+        escaped = escape_backslashes(value)
+
+        delimiter, escaped = escape_quotes_and_get_delimiter(escaped)
+
+        if escaped.include?("\n")
+          format_multiline_string(prefix, delimiter, escaped)
+        elsif escaped.length < Jekyll::L10n::Constants::PO_SHORT_LINE_LENGTH
+          "#{prefix} #{delimiter}#{escaped}#{delimiter}"
+        else
+          format_long_string(prefix, delimiter, escaped)
+        end
+      end
+
+      def self.escape_backslashes(value)
+        value.gsub("\\", "\\\\")
+      end
+
+      def self.escape_quotes_and_get_delimiter(escaped)
+        double_quote_count = escaped.count('"')
+        single_quote_count = escaped.count("'")
+        use_single_quotes = double_quote_count.positive? && single_quote_count.zero?
+
+        if use_single_quotes
+          ["'", escaped.gsub("'", "\\'")]
+        else
+          ['"', escaped.gsub('"', '\\"')]
+        end
+      end
+
+      def self.format_multiline_string(prefix, delimiter, escaped)
+        lines = ["#{prefix} #{delimiter}#{delimiter}"]
+        escaped.split("\n").each do |line|
+          next if line.strip.empty?
+
+          lines << "#{delimiter}#{line}#{delimiter}"
+        end
+        lines << "#{delimiter}#{delimiter}" if lines.length == 1
+        lines.join("\n")
+      end
+
+      def self.format_long_string(prefix, delimiter, escaped)
+        lines = ["#{prefix} #{delimiter}#{delimiter}"]
+        i = 0
+
+        while i < escaped.length
+          chunk = escaped[i...i + Jekyll::L10n::Constants::PO_LINE_LENGTH]
+          lines << "#{delimiter}#{chunk}#{delimiter}"
+          i += Jekyll::L10n::Constants::PO_LINE_LENGTH
+        end
+
+        lines.join("\n")
+      end
+    end
+  end
+end

data/lib/jekyll-l10n/translation/block_text_extractor.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative "../utils/text_normalizer"
+require_relative "../utils/html_text_utils"
+require_relative "../utils/html_elements"
+require_relative "../utils/text_validator"
+
+module Jekyll
+  module L10n
+    # Extracts normalized text from block-level HTML elements.
+    #
+    # BlockTextExtractor extracts the complete text content from a block element
+    # while removing nested block-level elements and empty icon tags. This is used
+    # to match against block-level translations where the entire element has a
+    # single translation rather than individual text node translations.
+    #
+    # Key responsibilities:
+    # * Extract text from extractable block elements
+    # * Remove nested block elements from text
+    # * Remove empty icon tags (external link markers)
+    # * Normalize and validate extracted text
+    # * Decode HTML entities
+    #
+    # @example
+    #   text = BlockTextExtractor.extract(paragraph_node)
+    #   # Returns normalized text from paragraph, useful for finding block translations
+    module BlockTextExtractor
+      extend self
+
+      # Extract normalized block text from an element.
+      #
+      # Returns nil if element is not extractable or if extracted text fails
+      # validation. Clones the node, removes nested block elements and empty
+      # icon tags, normalizes whitespace, decodes HTML entities, and validates.
+      #
+      # @param node [Nokogiri::XML::Element] DOM element to extract from
+      # @return [String, nil] Normalized text from element, or nil if not valid
+      def extract(node)
+        return nil unless extractable?(node)
+
+        clone = node.dup
+        HtmlTextUtils.remove_block_elements(clone)
+        HtmlTextUtils.remove_empty_icon_tags(clone)
+
+        text = TextNormalizer.normalize(clone.inner_html).strip
+        text = HtmlTextUtils.decode_html_entities(text)
+
+        TextValidator.valid?(text) ? text : nil
+      end
+
+      def extractable?(node)
+        node.element? && HtmlElements::CONTENT_ELEMENTS.include?(node.name)
+      end
+    end
+  end
+end

data/lib/jekyll-l10n/translation/html_translator.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+
+require_relative "../constants"
+require_relative "../utils/text_normalizer"
+require_relative "../utils/debug_logger"
+require_relative "../utils/translation_resolver"
+require_relative "../utils/url_transformer"
+require_relative "../utils/html_elements"
+require_relative "../utils/html_parser"
+require_relative "../utils/text_validator"
+require_relative "block_text_extractor"
+
+module Jekyll
+  module L10n
+    # Applies translations from PO files to HTML text nodes and DOM attributes.
+    #
+    # HtmlTranslator walks the DOM tree of parsed HTML documents and applies translations
+    # to text content and configurable HTML attributes (title, alt, aria-label, etc.).
+    # It supports three fallback modes for missing translations: using original text,
+    # marking untranslated content, or leaving blank. It also handles block-level
+    # translations for elements with complete translations and preserves URL transformations.
+    #
+    # Key responsibilities:
+    # * Parse full HTML documents while preserving DOCTYPE and structure
+    # * Translate text nodes using normalized text for matching
+    # * Translate HTML attributes (title, alt, aria-label, placeholder, aria-description)
+    # * Apply fallback modes when translations are missing (english/marker/empty)
+    # * Handle block-level translations for content elements
+    # * Transform relative URLs to locale-prefixed URLs
+    # * Remove auto-inserted meta charset tags from serialized HTML
+    #
+    # @example
+    #   translator = HtmlTranslator.new('english', ['title', 'alt'])
+    #   translated = translator.translate(html, translations, 'es', '/baseurl')
+    #   # Returns HTML with text and attributes translated to Spanish
+    #
+    # @see Jekyll::L10n::TranslationResolver for fallback mode logic
+    class HtmlTranslator
+      attr_reader :fallback_mode, :translatable_attrs, :debug_logging
+
+      # Initialize a new HtmlTranslator.
+      #
+      # @param fallback_mode [String, Symbol] How to handle missing translations:
+      #   'english' or :english - use original text (default)
+      #   'marker' or :marker - wrap with untranslated marker
+      #   'empty' or :empty - leave blank
+      # @param translatable_attrs [Array<String>] HTML attributes to extract and translate
+      #   (e.g., ['title', 'alt', 'aria-label', 'placeholder', 'aria-description'])
+      # @param debug_logging [Boolean] (keyword) Enable detailed debug logging for
+      #   translation process
+      def initialize(fallback_mode, translatable_attrs, debug_logging: false)
+        @fallback_mode = fallback_mode
+        @translatable_attrs = translatable_attrs
+        @debug_logging = debug_logging
+      end
+
+      # Translate an HTML document to a specific locale.
+      #
+      # Parses the HTML document, applies translations to text nodes and attributes,
+      # transforms URLs to be locale-aware, and returns the translated HTML with
+      # proper structure preserved.
+      #
+      # @param html [String] Full HTML document to translate
+      # @param translations [Hash] Translation hash mapping normalized text to translated strings
+      #   or metadata hashes with :msgstr, :reference, :fuzzy keys
+      # @param locale [String] Target locale code (defaults to "en"; e.g., 'es', 'fr')
+      # @param baseurl [String] Base URL for relative URL transformation (defaults to "")
+      # @return [String] Translated HTML with URLs transformed and meta charset removed
+      def translate(html, translations, locale = "en", baseurl = "")
+        # Use HtmlParser to properly parse full HTML documents while preserving
+        # DOCTYPE, html tag, and document structure. Any auto-inserted meta tags are
+        # removed by HtmlParser.remove_meta_charset after serialization.
+        # See: spec/regression/nokogiri_meta_tag_spec.rb for regression tests
+        doc = HtmlParser.parse_document(html)
+
+        translate_node(doc, translations)
+
+        # Transform URLs on the document object before serialization to avoid double-parsing
+        # and preserve the correct DOCTYPE and HTML structure. This prevents Nokogiri from
+        # downgrading to HTML 4.0 DOCTYPE when parsing the serialized HTML again.
+        # See: spec/jekyll-l10n/utils/url_transformer_spec.rb for tests
+        UrlTransformer.transform_document(doc, locale, baseurl)
+
+        result = doc.to_html
+
+        # Remove the auto-inserted meta tag by libxml2 during HTML serialization
+        # Matches: <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+        HtmlParser.remove_meta_charset(result)
+      end
+
+      private
+
+      def translate_node(node, translations)
+        return if node.nil?
+
+        translate_text_node(node, translations) if node.text? && node.parent
+        translate_node_attributes(node, translations) if node.element?
+        node.children.each { |child| translate_node(child, translations) }
+      end
+
+      def translate_text_node(node, translations)
+        original_content = node.content
+        text = original_content.strip
+        return if should_skip_translation?(text)
+
+        log_text_node_debug(text) if should_log_text_debug?(text)
+
+        normalized_text = TextNormalizer.normalize(text)
+        translated = TranslationResolver.resolve(node, normalized_text, translations)
+
+        return if apply_block_level_translation?(node, normalized_text, translated)
+
+        if @debug_logging
+          log_translation_debug_info(text, normalized_text, translated,
+                                     translations)
+        end
+
+        node.content = apply_fallback(original_content, translated)
+      end
+
+      def translate_node_attributes(node, translations)
+        @translatable_attrs.each do |attr_name|
+          value = node[attr_name]
+          next if value.nil? || value.empty?
+
+          value = value.strip
+          next unless TextValidator.valid?(value)
+
+          normalized_value = TextNormalizer.normalize(value)
+          translated = translations[normalized_value]
+          node[attr_name] = apply_fallback(value, translated)
+        end
+      end
+
+      def content_element?(node)
+        return false unless node.element?
+
+        HtmlElements::CONTENT_ELEMENTS.include?(node.name)
+      end
+
+      def apply_block_level_translation(parent, translation)
+        if @debug_logging
+          Jekyll.logger.info "Localization",
+                             "[HtmlTranslator] BLOCK-LEVEL TRANSLATION for <#{parent.name}>"
+          truncate_length = Jekyll::L10n::Constants::LOG_TRUNCATE_LONG
+          Jekyll.logger.info "Localization",
+                             "[HtmlTranslator]   Translation: #{translation[0..truncate_length]}..."
+        end
+
+        parent.children.each(&:remove)
+        parent.inner_html = translation
+      end
+
+      # Apply fallback when translation is missing or empty.
+      #
+      # === Three Fallback Modes ===
+      # 1. :english (default)
+      #    Returns original English text (msgid)
+      #    Best for: Production sites - untranslated content is still readable
+      #    Example: "Hello World" -> "Hello World" (if no translation found)
+      #
+      # 2. :marker
+      #    Wraps text with visible marker "[UNTRANSLATED: ...]"
+      #    Best for: QA/Development - clearly identifies missing translations
+      #    Example: "Hello World" -> "[UNTRANSLATED: Hello World]"
+      #    Why: Stakeholders can see exactly which content needs translation
+      #
+      # 3. :empty
+      #    Leaves completely blank
+      #    Best for: Templates where blank is preferred over wrong language
+      #    Example: "Hello World" -> ""
+      #    Why: Some designs handle missing content better than showing wrong language
+      #
+      # When might translation be empty but msgstr exist?
+      # - Translator marked entry as fuzzy (incomplete)
+      # - Entry has msgstr: "" (deliberately blank translation)
+      def apply_fallback(msgid, msgstr)
+        return msgstr if msgstr && !msgstr.empty?
+
+        case @fallback_mode
+        when Jekyll::L10n::Constants::FALLBACK_MODE_MARKER, :marker
+          "#{Jekyll::L10n::Constants::UNTRANSLATED_MARKER} #{msgid}"
+        when Jekyll::L10n::Constants::FALLBACK_MODE_EMPTY, :empty
+          ""
+        else
+          msgid
+        end
+      end
+
+      def log_text_node_debug(text)
+        truncate_length = Jekyll::L10n::Constants::LOG_TRUNCATE_MEDIUM
+        message = "[HtmlTranslator] Processing text node: " \
+                  "#{text[0..truncate_length]}... (#{text.length} chars)"
+        Jekyll.logger.info "Localization", message
+      end
+
+      def should_skip_translation?(text)
+        !TextValidator.valid?(text)
+      end
+
+      def should_log_text_debug?(text)
+        @debug_logging && text.include?("attribute")
+      end
+
+      def apply_block_level_translation?(node, normalized_text, translated)
+        return false unless translated && node.parent && content_element?(node.parent)
+
+        return false if TranslationResolver.contains_protected_elements?(node.parent)
+
+        block_text = BlockTextExtractor.extract(node.parent)
+        return false unless block_text && block_text != normalized_text
+
+        apply_block_level_translation(node.parent, translated)
+        true
+      end
+
+      def log_translation_debug_info(text, normalized_text, translated, translations)
+        translation_data = DebugLogger::TranslationData.new(:text            => text,
+                                                            :normalized_text => normalized_text,
+                                                            :translated      => translated,
+                                                            :translations    => translations)
+        DebugLogger.log_translation_details(self, translation_data)
+      end
+
+      private :log_text_node_debug, :should_skip_translation?, :should_log_text_debug?,
+              :apply_block_level_translation?, :log_translation_debug_info
+    end
+  end
+end