jekyll-l10n 1.0.5

data/lib/jekyll-l10n/po_file/reader.rb

@@ -0,0 +1,518 @@
+# frozen_string_literal: true
+
+require "gettext/po"
+require_relative "../utils/file_operations"
+
+module Jekyll
+  module L10n
+    # Parses GNU Gettext PO files into translation hashes.
+    #
+    # PoFileReader reads and parses PO files in standard GNU Gettext format, supporting
+    # multiple parsing modes: simple (msgid -> msgstr), with references (for debugging),
+    # and with merge metadata (including fuzzy flags). It handles multi-line strings,
+    # comment extraction, and various escape sequences. Both instance-based and
+    # class-based APIs are supported for backward compatibility.
+    #
+    # Key responsibilities:
+    # * Parse PO files into translation hashes
+    # * Handle msgid/msgstr pairs with continuation lines
+    # * Extract and preserve reference comments (file location references)
+    # * Extract and preserve fuzzy flags during merging
+    # * Parse multi-line strings with proper escaping
+    # * Support three modes: simple, with_references, and for_merge
+    # * Handle both file paths and inline content strings
+    #
+    # @example
+    #   reader = PoFileReader.new('_locales/es.po')
+    #   simple = reader.parse  # { "msgid" => "msgstr" }
+    #   with_refs = reader.parse_with_references  # { "msgid" => { msgstr: "...", reference:
+    #     "..." } }
+    #   for_merge = reader.parse_for_merge  # { "msgid" => { msgstr: "...", reference:
+    #     "...", fuzzy: false } }
+    # rubocop:disable Metrics/ClassLength
+    class PoFileReader
+      # rubocop:enable Metrics/ClassLength
+      MSGID_PATTERN = %r!^msgid ['"](.*)['"] *$!.freeze unless const_defined?(:MSGID_PATTERN)
+      MSGSTR_PATTERN = %r!^msgstr ['"](.*)['"] *$!.freeze unless const_defined?(:MSGSTR_PATTERN)
+      NO_REFERENCE = nil unless const_defined?(:NO_REFERENCE)
+
+      # Initialize a new PoFileReader.
+      #
+      # Accepts either a file path (if file exists) or inline PO content. Determines
+      # which based on whether the path exists in the filesystem. Defaults to nil,
+      # which initializes the reader with empty content.
+      #
+      # @param po_path_or_content [String, nil] File path to PO file or inline content
+      #   string (defaults to nil)
+      def initialize(po_path_or_content = nil)
+        # Support both file path and content string
+        if po_path_or_content && File.exist?(po_path_or_content.to_s)
+          @po_path = po_path_or_content
+          @content = nil
+        else
+          @content = po_path_or_content
+          @po_path = nil
+        end
+      end
+
+      # Parse PO file into simple translation hash.
+      #
+      # Returns hash mapping msgid strings to msgstr strings (no metadata).
+      #
+      # @return [Hash] Simple translation hash { msgid => msgstr }
+      def parse
+        content = load_content
+        process_po_lines_instance(content, false)
+      end
+
+      # Parse PO file with reference comments preserved.
+      #
+      # Returns hash mapping msgid strings to metadata hashes containing msgstr
+      # and reference (file location for debugging).
+      #
+      # @return [Hash] Translation hash with references { msgid => { msgstr: "...",
+      #   reference: "..." } }
+      def parse_with_references
+        content = load_content
+        process_po_lines_instance(content, true)
+      end
+
+      # Parse PO file with all metadata for merging.
+      #
+      # Returns hash mapping msgid strings to metadata hashes containing msgstr,
+      # reference, and fuzzy flag. Used when merging with existing translations.
+      #
+      # @return [Hash] Translation hash with merge metadata { msgid => { msgstr: "...",
+      #   reference: "...", fuzzy: false } }
+      def parse_for_merge
+        content = load_content
+        process_po_lines_instance(content, :merge)
+      end
+
+      # Parse a PO file (class method, for backward compatibility).
+      #
+      # @param po_path [String] Path to PO file
+      # @return [Hash] Simple translation hash
+      def self.parse(po_path)
+        new(po_path).parse
+      end
+
+      # Parse a PO file with references (class method, for backward compatibility).
+      #
+      # @param po_path [String] Path to PO file
+      # @return [Hash] Translation hash with references
+      def self.parse_with_references(po_path)
+        new(po_path).parse_with_references
+      end
+
+      # Parse a PO file for merging (class method, for backward compatibility).
+      #
+      # @param po_path [String] Path to PO file
+      # @return [Hash] Translation hash with merge metadata
+      def self.parse_for_merge(po_path)
+        new(po_path).parse_for_merge
+      end
+
+      # Backward compatibility wrapper
+      def self.process_po_lines(content)
+        process_po_lines_internal(content, false)
+      end
+
+      # Backward compatibility wrapper
+      def self.process_line(lines, idx, translations)
+        process_line_internal(lines, idx, translations, false)
+      end
+
+      def self.msgid_line?(line)
+        line.start_with?("msgid ")
+      end
+
+      # Splits PO file content into individual lines
+      def self.split_lines(content)
+        content.split("\n")
+      end
+
+      # Skips blank lines and comments before processing entries
+      def self.skip_blank_and_comments(lines, idx)
+        return idx if idx >= lines.length
+
+        line = lines[idx].strip
+        if line.empty? || line.start_with?("#")
+          idx + 1
+        else
+          idx
+        end
+      end
+
+      # Unified method for processing msgid/msgstr pairs
+      # with optional reference and fuzzy metadata
+      # with_mode: false (default, simple format), true (with reference), :merge (with both)
+      # rubocop:disable Metrics/ParameterLists, Metrics/AbcSize, Metrics/PerceivedComplexity
+      def self.process_msgid_msgstr_pair(lines, start_idx, translations,
+                                         reference: nil, fuzzy: nil, with_mode: false)
+        # rubocop:enable Metrics/ParameterLists, Metrics/AbcSize, Metrics/PerceivedComplexity
+        # Handle nil sentinel values (from NO_REFERENCE constant)
+        reference = nil if reference == NO_REFERENCE
+        fuzzy = nil if fuzzy == NO_REFERENCE
+
+        msgid = extract_msgid_and_continuation(lines, start_idx)
+        msgid_value = msgid[:value]
+        i = msgid[:next_line]
+
+        if i < lines.length && lines[i].strip.start_with?("msgstr ")
+          msgstr = extract_msgstr_and_continuation(lines, i)
+          msgstr_value = msgstr[:value]
+          i = msgstr[:next_line]
+
+          # Always use metadata format if in reference or merge mode
+          with_metadata = with_mode == true || with_mode == :merge || !reference.nil? || !fuzzy.nil?
+          store_translation(
+            translations, msgid_value, msgstr_value,
+            :reference => reference, :fuzzy => fuzzy, :with_metadata => with_metadata
+          )
+        else
+          i += 1
+        end
+
+        i
+      end
+
+      # Backward compatibility alias
+      # rubocop:disable Metrics/ParameterLists
+      def self.process_msgid_msgstr_pair_with_metadata(lines, start_idx, translations,
+                                                       reference: nil, fuzzy: nil)
+        # rubocop:enable Metrics/ParameterLists
+        process_msgid_msgstr_pair(lines, start_idx, translations, :reference => reference,
+:fuzzy => fuzzy, :with_mode => :merge)
+      end
+
+      # Backward compatibility alias
+      def self.process_msgid_msgstr_pair_with_reference(lines, start_idx, translations, reference)
+        process_msgid_msgstr_pair(
+          lines, start_idx, translations,
+          :reference => reference, :fuzzy => nil, :with_mode => true
+        )
+      end
+
+      # Backward compatibility alias
+      # rubocop:disable Metrics/ParameterLists
+      def self.process_msgid_msgstr_pair_internal(lines, start_idx, translations,
+                                                  reference = nil, fuzzy = nil)
+        # rubocop:enable Metrics/ParameterLists
+        process_msgid_msgstr_pair(lines, start_idx, translations, :reference => reference,
+                                                                  :fuzzy     => fuzzy)
+      end
+
+      def self.process_po_lines_with_references(content)
+        process_po_lines_internal(content, true)
+      end
+
+      def self.process_po_lines_for_merge(content)
+        process_po_lines_internal(content, :merge)
+      end
+
+      def self.process_po_lines_internal(content, with_references)
+        translations = {}
+        lines = split_lines(content)
+        i = 0
+
+        while i < lines.length
+          i = case with_references
+              when true
+                process_line_internal(lines, i, translations, true)
+              when :merge
+                process_line_internal(lines, i, translations, :merge)
+              else
+                process_line_internal(lines, i, translations, false)
+              end
+        end
+
+        translations
+      end
+
+      # Backward compatibility wrappers
+      def self.process_line_with_reference(lines, idx, translations)
+        process_line_internal(lines, idx, translations, true)
+      end
+
+      def self.process_line_for_merge(lines, idx, translations)
+        process_line_internal(lines, idx, translations, :merge)
+      end
+
+      def self.process_line_internal(lines, idx, translations, with_references)
+        return idx + 1 if idx >= lines.length
+
+        idx = skip_blank_and_comments(lines, idx)
+        return idx if idx >= lines.length
+
+        line = lines[idx].strip
+
+        if msgid_line?(line)
+          process_msgid_with_references(lines, idx, translations, with_references)
+        else
+          idx + 1
+        end
+      end
+
+      def self.process_msgid_with_references(lines, idx, translations, with_references)
+        case with_references
+        when true
+          reference = extract_reference_before_msgid(lines, idx)
+          process_msgid_msgstr_pair(
+            lines, idx, translations,
+            :reference => reference, :fuzzy => nil, :with_mode => true
+          )
+        when :merge
+          reference, fuzzy = extract_reference_and_fuzzy_before_msgid(lines, idx)
+          process_msgid_msgstr_pair(
+            lines, idx, translations,
+            :reference => reference, :fuzzy => fuzzy, :with_mode => :merge
+          )
+        else
+          process_msgid_msgstr_pair(lines, idx, translations, :reference => nil, :fuzzy => nil,
+:with_mode => false)
+        end
+      end
+
+      # Unified metadata extraction: extracts reference and optionally fuzzy flag
+      def self.extract_metadata_before_msgid(lines, msgid_idx, include_fuzzy: false)
+        reference = nil
+        fuzzy = false
+        comments_end = msgid_idx - 1
+
+        while comments_end >= 0
+          comment_line = lines[comments_end].strip
+          break unless comment_line.start_with?("#") || comment_line.empty?
+
+          reference = extract_reference_from_line(comment_line) || reference
+          fuzzy = true if include_fuzzy && fuzzy_line?(comment_line)
+
+          comments_end -= 1
+        end
+
+        include_fuzzy ? [reference, fuzzy] : reference
+      end
+
+      def self.extract_reference_from_line(comment_line)
+        comment_line.sub(%r!^#:\s*!, "").strip if comment_line.start_with?("#:")
+      end
+
+      def self.fuzzy_line?(comment_line)
+        comment_line.start_with?("#,") && comment_line.include?("fuzzy")
+      end
+
+      # Backward compatibility wrapper
+      def self.extract_reference_before_msgid(lines, msgid_idx)
+        extract_metadata_before_msgid(lines, msgid_idx, :include_fuzzy => false)
+      end
+
+      # Backward compatibility wrapper
+      def self.extract_reference_and_fuzzy_before_msgid(lines, msgid_idx)
+        extract_metadata_before_msgid(lines, msgid_idx, :include_fuzzy => true)
+      end
+
+      def self.extract_msgid_and_continuation(lines, start_idx)
+        extract_po_field(lines, start_idx, MSGID_PATTERN)
+      end
+
+      def self.extract_msgstr_and_continuation(lines, start_idx)
+        extract_po_field(lines, start_idx, MSGSTR_PATTERN)
+      end
+
+      def self.extract_po_field(lines, start_idx, pattern)
+        line = lines[start_idx].strip
+
+        match = line.match(pattern)
+        values = match ? [match[1]] : []
+        delimiter = line.include?("'") ? "'" : '"'
+
+        collect_continuation_lines(lines, start_idx + 1, values, delimiter)
+      end
+
+      def self.collect_continuation_lines(lines, start_idx, values, delimiter)
+        idx = start_idx
+
+        while idx < lines.length
+          cont_line = lines[idx].strip
+
+          break if stop_collecting?(cont_line)
+
+          break unless continuation_line?(cont_line)
+
+          unescaped = unescape_string(cont_line[1...-1], delimiter)
+          values << unescaped
+          idx += 1
+
+        end
+
+        combined_value = values.join
+
+        { :value => combined_value, :next_line => idx }
+      end
+
+      def self.continuation_line?(line)
+        (line.start_with?('"') && line.end_with?('"')) ||
+          (line.start_with?("'") && line.end_with?("'"))
+      end
+
+      # Unescape PO file string values containing escape sequences.
+      #
+      # === IMPORTANT: Order Matters for Correctness ===
+      # Must unescape escaped quotes BEFORE unescaping backslashes.
+      # Why: If we unescape backslashes first, we lose information about which
+      # backslashes were part of escape sequences vs. literal text.
+      #
+      # Example with wrong order (backslash first):
+      #   Input: "Say \\" Hello"  (should be: Say \ Hello with closing quote)
+      #   Wrong: "\\\\" -> "\\" -> Remove quotes -> "Say \" Hello" (quote not closed!)
+      #
+      # Correct order (quote first):
+      #   Input: "Say \\" Hello"
+      #   Right: "\\" -> " " (two backslashes become one literal backslash) -> "Say \ Hello"
+      #
+      # Single vs Double quotes matter:
+      # - Single quotes: Use \\' and \\\\
+      # - Double quotes: Use \\" and \\\\
+      def self.unescape_string(str, delimiter)
+        if delimiter == "'"
+          str.gsub("\\'", "'").gsub("\\\\", "\\")
+        else
+          str.gsub('\\"', '"').gsub("\\\\", "\\")
+        end
+      end
+
+      def self.read_po_file(po_path)
+        FileOperations.read_utf8(po_path)
+      end
+
+      # Store translation with optional metadata (reference location and fuzzy flag).
+      #
+      # === Three Storage Formats ===
+      # 1. Simple: { msgid => msgstr }
+      #    Used during translation lookup - minimal memory, fastest access
+      #
+      # 2. With Reference: { msgid => { msgstr: "...", reference: "file.html:10" } }
+      #    Used during extraction - preserves source location for debugging
+      #
+      # 3. With Merge Metadata: { msgid => { msgstr: "...", reference: "...", fuzzy: false } }
+      #    Used during merging - tracks fuzzy flag for incomplete translations
+      #
+      # Why three formats instead of one?
+      # - Memory efficiency: Simple format used most often (translation lookup)
+      # - Flexibility: Can handle different parsing modes without wasting storage
+      # - Backward compatibility: Supports legacy calling conventions
+      # rubocop:disable Metrics/ParameterLists
+      def self.store_translation(translations, msgid, msgstr, reference: nil, fuzzy: nil,
+                                 with_metadata: false)
+        # rubocop:enable Metrics/ParameterLists
+        return if msgid.nil? || msgstr.nil? || msgid.empty?
+
+        translations[msgid] = build_translation_entry(msgstr, reference, fuzzy, with_metadata)
+      end
+
+      def self.build_translation_entry(msgstr, reference, fuzzy, with_metadata)
+        # Simple format when no metadata requested and none provided
+        return msgstr if !with_metadata && reference.nil? && fuzzy.nil?
+
+        # Build metadata hash based on what's provided
+        entry = { :msgstr => msgstr }
+        entry[:reference] = reference unless reference.nil?
+        entry[:fuzzy] = fuzzy unless fuzzy.nil?
+        entry[:comment] = nil if !fuzzy.nil? || !reference.nil?
+        entry
+      end
+
+      # Kept for backward compatibility with existing tests
+      # Supports both positional and keyword argument calling styles
+      # rubocop:disable Metrics/ParameterLists
+      def self.store_translation_internal(translations, msgid, msgstr, reference: nil, fuzzy: nil)
+        # rubocop:enable Metrics/ParameterLists
+        # Handle nil sentinel values (from NO_REFERENCE constant)
+        reference = nil if reference == NO_REFERENCE
+        fuzzy = nil if fuzzy == NO_REFERENCE
+        store_translation(translations, msgid, msgstr, :reference => reference, :fuzzy => fuzzy)
+      end
+
+      # Backward compatibility wrappers for old method signatures
+      def self.store_translation_with_reference(translations, msgid, msgstr, reference:)
+        store_translation(translations, msgid, msgstr, :reference => reference, :fuzzy => nil,
+                         :with_metadata => true)
+      end
+
+      def self.store_translation_with_fuzzy(translations, msgid, msgstr, fuzzy:)
+        store_translation(translations, msgid, msgstr, :reference => nil, :fuzzy => fuzzy,
+                         :with_metadata => true)
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def self.store_translation_with_reference_and_fuzzy(translations, msgid, msgstr,
+                                                          reference:, fuzzy:)
+        # rubocop:enable Metrics/ParameterLists
+        store_translation(translations, msgid, msgstr, :reference => reference, :fuzzy => fuzzy,
+                         :with_metadata => true)
+      end
+
+      # Determines if we should stop collecting continuation lines
+      def self.stop_collecting?(line)
+        line.empty? ||
+          line.start_with?("#") ||
+          line.start_with?("msgid") ||
+          line.start_with?("msgstr")
+      end
+
+      private
+
+      def load_content
+        @content || self.class.read_po_file(@po_path)
+      end
+
+      def process_po_lines_instance(content, with_references)
+        translations = {}
+        lines = self.class.split_lines(content)
+        i = 0
+
+        while i < lines.length
+          i = case with_references
+              when true
+                process_line_instance(lines, i, translations, true)
+              when :merge
+                process_line_instance(lines, i, translations, :merge)
+              else
+                process_line_instance(lines, i, translations, false)
+              end
+        end
+
+        translations
+      end
+
+      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      def process_line_instance(lines, idx, translations, with_references)
+        # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+        return idx + 1 if idx >= lines.length
+
+        idx = self.class.skip_blank_and_comments(lines, idx)
+        return idx if idx >= lines.length
+
+        line = lines[idx].strip
+
+        if self.class.msgid_line?(line)
+          case with_references
+          when true
+            reference = self.class.extract_reference_before_msgid(lines, idx)
+            self.class.process_msgid_msgstr_pair(lines, idx, translations, :reference => reference,
+:fuzzy => nil, :with_mode => true)
+          when :merge
+            reference, fuzzy = self.class.extract_reference_and_fuzzy_before_msgid(lines, idx)
+            self.class.process_msgid_msgstr_pair(lines, idx, translations, :reference => reference,
+:fuzzy => fuzzy, :with_mode => :merge)
+          else
+            self.class.process_msgid_msgstr_pair(lines, idx, translations, :reference => nil,
+:fuzzy => nil, :with_mode => false)
+          end
+        else
+          idx + 1
+        end
+      end
+    end
+  end
+end