jekyll-l10n 1.7.1 → 3.0.0

data/lib/jekyll-l10n/extraction/compendium_merger.rb

@@ -4,7 +4,6 @@ require 'fileutils'
 require_relative '../po_file/manager'
 require_relative '../po_file/fuzzy_matcher'
 require_relative '../utils/page_locales_config'
-require_relative '../utils/site_config_accessor'
 
 module Jekyll
   module L10n
@@ -35,8 +34,6 @@ module Jekyll
       # @param site [Jekyll::Site] Jekyll site object
       def initialize(site)
         @site = site
-        with_locales_data = SiteConfigAccessor.extract_locales_data(@site)
-        @site_config = PageLocalesConfig.new({ 'with_locales_data' => with_locales_data })
       end
 
       # Merge page-specific PO files into compendia for all locales.
@@ -49,8 +46,11 @@ module Jekyll
       # @param config [PageLocalesConfig] Localization configuration with locales list
       # @return [void]
       def merge_compendia(po_manager, config)
+        # Fuzzy scores depend only on the two English strings, and every
+        # locale compares the same new strings, so they are scored once per run.
+        @fuzzy_scores = {}
         config.locales.each do |locale|
-          process_locale(locale, po_manager, config)
+          process_locale(locale, po_manager)
         end
       end
 
@@ -58,9 +58,8 @@ module Jekyll
 
       # Process a single locale: merge existing compendium with new page translations
       # Skips writing when translations are unchanged to avoid unnecessary file rewrites
-      def process_locale(locale, po_manager, config)
-        compendium_path = File.join(@site.source, config.locales_dir, "#{locale}.po")
-        existing_compendium = load_existing_compendium(compendium_path)
+      def process_locale(locale, po_manager)
+        existing_compendium = load_existing_compendium(po_manager.compendium_path(locale))
         merged = po_manager.merge_po_files(locale)
 
         combined = build_combined_hash(existing_compendium)
@@ -70,7 +69,7 @@ module Jekyll
           combined_entries = format_compendium_entries(combined)
           po_manager.save_compendium(locale, combined_entries)
         end
-        cleanup_locale_directory(locale, config)
+        cleanup_locale_directory(po_manager.locale_dir(locale))
       end
 
       # Check if compendium content is unchanged (same msgids and msgstrs)
@@ -146,7 +145,7 @@ module Jekyll
       end
 
       def resolve_new_entry(combined, msgid, entry, fuzzy_candidates, matched_old_msgids)
-        match = PoFuzzyMatcher.find_match(msgid, fuzzy_candidates)
+        match = PoFuzzyMatcher.find_match(msgid, fuzzy_candidates, scores: @fuzzy_scores)
         if match
           combined[msgid] = create_fuzzy_entry(entry, match)
           matched_old_msgids << match[:msgid]
@@ -195,8 +194,7 @@ module Jekyll
 
       # IMPORTANT: Remove locale-specific directory after compendium merge
       # This cleans up page-specific PO files that have been merged into the compendium
-      def cleanup_locale_directory(locale, config)
-        locale_dir = File.join(@site.source, config.locales_dir, locale)
+      def cleanup_locale_directory(locale_dir)
         FileUtils.rm_rf(locale_dir) if File.directory?(locale_dir)
       end
 

data/lib/jekyll-l10n/extraction/compendium_translator.rb

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 
+require_relative '../constants'
+require_relative '../instrumentation'
 require_relative '../po_file/manager'
 require_relative '../utils/page_locales_config'
 require_relative '../utils/po_entry_converter'
-require_relative '../utils/site_config_accessor'
 require_relative '../utils/logger_formatter'
 require_relative '../translation/libre_translator'
 
@@ -33,18 +34,19 @@ module Jekyll
       # @param site [Jekyll::Site] Jekyll site object
       def initialize(site)
         @site = site
-        with_locales_data = SiteConfigAccessor.extract_locales_data(@site)
-        @site_config = PageLocalesConfig.new({ 'with_locales_data' => with_locales_data })
       end
 
       # Translate compendia for all configured locales.
       #
       # Checks if LibreTranslate is enabled. If so, for each configured locale,
       # loads the compendium PO file, identifies untranslated entries, sends them
-      # to LibreTranslate API, and saves the updated file.
+      # to LibreTranslate API, and saves the updated file. Locales are
+      # translated at once; their requests share libretranslate_concurrency.
       #
       # @param config [PageLocalesConfig] Localization configuration
       # @return [void]
+      # @raise [LibreTranslator::TranslationError] If a locale failed, once
+      #   every locale has finished; the locales that finished are saved
       def translate_compendia(config)
         po_manager = PoFileManager.new(@site, config.locales_dir)
         translate_compendia_for_locale(po_manager, config)
@@ -59,15 +61,29 @@ module Jekyll
         log_translation_start(config)
         translator = LibreTranslator.new(config)
 
-        config.locales.each do |locale|
-          process_single_locale(locale, config, translator, po_manager)
+        threads = config.locales.map do |locale|
+          Instrumentation.traced_thread do
+            Thread.current.report_on_exception = false
+            process_single_locale(locale, translator, po_manager)
+          end
         end
+        join_locales(threads)
 
         log_translation_complete(config)
       end
 
-      def process_single_locale(locale, config, translator, po_manager)
-        compendium_path = File.join(@site.source, config.locales_dir, "#{locale}.po")
+      def join_locales(threads)
+        first_error = nil
+        threads.each do |thread|
+          thread.join
+        rescue StandardError => e
+          first_error ||= e
+        end
+        raise first_error if first_error
+      end
+
+      def process_single_locale(locale, translator, po_manager)
+        compendium_path = po_manager.compendium_path(locale)
         LoggerFormatter.debug_if_enabled('CompendiumTranslator',
                                          "Processing compendium file: #{compendium_path}")
         return unless File.exist?(compendium_path)
@@ -119,17 +135,17 @@ module Jekyll
 
       def log_translation_start(config)
         locales = config.locales.join(', ')
-        Jekyll.logger.info 'Localization',
+        Jekyll.logger.info Constants::LOG_PREFIX,
                            "Starting LibreTranslate translation for locales: #{locales}"
       end
 
       def log_translation_complete(config)
         locales = config.locales.join(', ')
-        Jekyll.logger.info 'Localization',
+        Jekyll.logger.info Constants::LOG_PREFIX,
                            "LibreTranslate translation complete for locales: #{locales}"
       end
 
-      private :translate_compendia_for_locale, :process_single_locale,
+      private :translate_compendia_for_locale, :join_locales, :process_single_locale,
               :log_compendia_enabled_check, :log_compendium_stats,
               :log_translation_complete_for_locale, :log_compendium_saved,
               :count_empty_entries, :log_translation_start, :log_translation_complete

data/lib/jekyll-l10n/extraction/config_loader.rb

@@ -1,24 +1,18 @@
 # frozen_string_literal: true
 
+require_relative '../configuration'
 require_relative '../utils/page_locales_config'
 require_relative '../utils/locale_url_formatter'
 require_relative '../utils/site_config_accessor'
 
 module Jekyll
   module L10n
-    # Loads and validates extraction configuration for files during build.
+    # Finds the settings for a built HTML file during extraction.
     #
-    # ExtractionConfigLoader finds extraction configuration for generated HTML
-    # files by matching them against Jekyll site pages. It validates whether
-    # extraction is enabled and configured for a file, loads page-specific
-    # settings, and identifies files to skip (localized page variants).
-    #
-    # Key responsibilities:
-    # * Match generated HTML files to Jekyll site pages
-    # * Validate that extraction is enabled for a file
-    # * Load page-specific extraction configuration
-    # * Identify localized page variants to skip
-    # * Extract CSS selectors for element exclusion
+    # Extraction works on output files, but settings belong to pages. This
+    # class matches an output file back to its source page and returns that
+    # page's merged settings. It also tells the Extractor which files to skip:
+    # the localized copies under locale folders such as `es/`.
     #
     # @example
     #   loader = ExtractionConfigLoader.new(site, '_site')
@@ -26,22 +20,16 @@ module Jekyll
     #     config = loader.load_page_config(file_path)
     #   end
     class ExtractionConfigLoader
-      # Initialize a new ExtractionConfigLoader.
-      #
-      # @param site [Jekyll::Site] Jekyll site object
-      # @param dest [String] Destination build directory
+      # @param site [Jekyll::Site] The Jekyll site
+      # @param dest [String] The site destination directory
       def initialize(site, dest)
         @site = site
         @dest = dest
       end
 
-      # Check if a file is valid for extraction.
-      #
-      # Matches the file to a Jekyll page and verifies extraction is enabled
-      # for that page in the configuration.
-      #
-      # @param file_path [String] Path to file to check
-      # @return [Boolean] True if file matches a page with extraction enabled
+      # @param file_path [String] Path of a built HTML file
+      # @return [Boolean] true if the file comes from a localized page with at
+      #   least one locale and `extract_on_build` not set to false
       def valid_for_extraction?(file_path)
         page_config = find_page_config_for_file(file_path)
         return false unless page_config
@@ -50,50 +38,37 @@ module Jekyll
         config.enabled? && config.extract_on_build?
       end
 
-      # Load extraction configuration for a file.
-      #
-      # Finds the Jekyll page matching this file and returns its localization
-      # configuration wrapped in PageLocalesConfig.
-      #
-      # @param file_path [String] Path to file
-      # @return [PageLocalesConfig] Localization configuration for the page
+      # @param file_path [String] Path of a built HTML file
+      # @return [PageLocalesConfig] Settings of the page that produced the file
       def load_page_config(file_path)
         page_config = find_page_config_for_file(file_path)
         PageLocalesConfig.new(page_config)
       end
 
-      # Find page configuration for a file.
-      #
-      # Matches a generated file path to a Jekyll page with extraction enabled.
-      # Returns the page's front matter data, or nil if no match found.
+      # Finds the localized page whose output is this file and returns its data
+      # with merged `with_locales_data` (see SiteConfigAccessor.effective_page_data).
       #
-      # @param file_path [String] Path to generated file
-      # @return [Hash, nil] Page front matter data if found, nil otherwise
+      # @param file_path [String] Path of a built HTML file
+      # @return [Hash, nil] The page data, or nil when no localized page matches
       def find_page_config_for_file(file_path)
         @site.pages.each do |page|
-          next unless page.data['with_locales'] == true
+          next unless Configuration.localizable?(page)
 
           page_output = page.output_ext ? page.destination('') : page.destination('/')
           next unless file_path.end_with?(page_output.sub(%r{/$}, '/index.html'))
 
-          return page.data
+          return SiteConfigAccessor.effective_page_data(page)
         end
 
         nil
       end
 
-      # Check if a file is a localized page variant that should be skipped.
-      #
-      # Localized pages are generated copies in locale subdirectories that
-      # shouldn't be re-extracted (extraction happens on original pages only).
-      # Since commit 3d2d4cd, country-subtag locales (e.g. es_ES) are written
-      # to BCP 47 hyphenated, lowercased URL directories (e.g. es-es/). Each
-      # configured locale is normalized through LocaleUrlFormatter.to_url_segment
-      # before comparison so both language-only (es) and country-subtag
-      # (es_ES → es-es) locales are correctly recognized.
+      # Localized copies are built from the English pages, so we extract only
+      # from the English files. Locale folders use the URL form of the locale
+      # code (es_ES becomes es-es), so compare against that form.
       #
-      # @param file_path [String] Path to file to check
-      # @return [Boolean] True if file is in a locale subdirectory
+      # @param file_path [String] Path of a built HTML file
+      # @return [Boolean] true if the file is inside a locale folder
       def skip_localized_page?(file_path)
         relative_path = file_path.sub(@dest, '')
 
@@ -101,16 +76,11 @@ module Jekyll
         all_locales.any? { |locale| relative_path.start_with?("/#{LocaleUrlFormatter.to_url_segment(locale)}/") }
       end
 
-      # Extract CSS selectors for element exclusion from configuration.
-      #
-      # Returns CSS selectors of elements to exclude from extraction (e.g.,
-      # script, style, code blocks). Defaults to sensible defaults if not configured.
-      #
-      # @param config [Hash] Page front matter data
-      # @return [Array<String>] CSS selectors for excluded elements
+      # @param config [PageLocalesConfig] The page's settings
+      # @return [Array<String>] CSS selectors of elements to leave out of
+      #   extraction (see PageLocalesConfig#exclude_selectors)
       def extract_exclude_selectors(config)
-        config.data.dig('with_locales_data', 'extraction', 'exclude_selectors') ||
-          ['script', 'style', 'code.language-plaintext', 'pre code']
+        config.exclude_selectors
       end
     end
   end

data/lib/jekyll-l10n/extraction/coverage_report.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require_relative '../constants'
+require_relative '../errors'
+require_relative '../po_file/path_builder'
+require_relative '../po_file/reader'
+require_relative '../utils/msgstr_validator'
+
+module Jekyll
+  module L10n
+    # Counts how much of each locale's compendium is translated.
+    #
+    # Each entry counts once, as translated, fuzzy, untranslated, or invalid
+    # (a msgstr whose markup MsgstrValidator rejects, so pages use the
+    # fallback). Extractor warns about each invalid entry, prints the counts
+    # with the extraction summary, and fails the build when a locale is below
+    # `translation.min_coverage`.
+    #
+    # Only entries whose msgid this build extracted count: a compendium also
+    # keeps entries for text the site no longer has. The counts come from the
+    # compendium files as they are after extraction, so with
+    # `update_compendium: false` they leave out strings no compendium holds
+    # yet.
+    #
+    # @example
+    #   report = CoverageReport.new(site, SiteConfigAccessor.site_config(site, all_locales: true),
+    #                               extracted_msgids)
+    #   report.warn_invalid
+    #   report.log
+    #   report.enforce!(90)  # raises Errors::CoverageError below 90%
+    class CoverageReport
+      # Entry counts for one locale.
+      Counts = Struct.new(:locale, :translated, :fuzzy, :untranslated, :invalid) do
+        # @return [Integer] Entries in the compendium
+        def total
+          translated + fuzzy + untranslated + invalid
+        end
+
+        # @return [Float] Translated entries as a percentage; 100 when the
+        #   compendium is empty
+        def percent
+          total.zero? ? 100.0 : translated * 100.0 / total
+        end
+
+        # @return [String] e.g. "fr: 92.5% translated (1200 of 1297; 40 fuzzy, 50 untranslated, 7 invalid)"
+        def to_s
+          "#{locale}: #{percent.round(1)}% translated (#{translated} of #{total}; " \
+            "#{fuzzy} fuzzy, #{untranslated} untranslated, #{invalid} invalid)"
+        end
+      end
+
+      # @param site [Jekyll::Site]
+      # @param config [PageLocalesConfig] The site config, for locales and locales_dir
+      # @param msgids [Set<String>] The msgids this build extracted
+      def initialize(site, config, msgids)
+        @site = site
+        @config = config
+        @msgids = msgids
+        @invalid = Hash.new { |hash, locale| hash[locale] = [] }
+      end
+
+      # @return [Array<Counts>] One entry per locale that has a compendium
+      def counts
+        @counts ||= @config.locales.filter_map do |locale|
+          path = PoPathBuilder.build(@site.source, @config.locales_dir, locale, nil)
+          count(locale, PoFileReader.parse_for_merge(path)) if File.exist?(path)
+        end
+      end
+
+      # Warn about each entry whose msgstr markup does not match its msgid.
+      #
+      # @return [void]
+      def warn_invalid
+        counts.each do |locale_counts|
+          @invalid[locale_counts.locale].each do |msgid|
+            Jekyll.logger.warn Constants::LOG_PREFIX,
+                               "#{locale_counts.locale}: msgstr markup does not match its msgid, " \
+                               "so pages show the fallback: #{msgid[0..Constants::LOG_TRUNCATE_LONG]}"
+          end
+        end
+      end
+
+      # Print one line per locale.
+      #
+      # @return [void]
+      def log
+        return if counts.empty?
+
+        Jekyll.logger.info Constants::LOG_PREFIX, 'Translation coverage:'
+        counts.each { |locale_counts| Jekyll.logger.info Constants::LOG_PREFIX, "  #{locale_counts}" }
+      end
+
+      # @param minimum [Integer, nil] Required percentage, or nil for none
+      # @return [void]
+      # @raise [Errors::CoverageError] When any locale is below the minimum
+      def enforce!(minimum)
+        return if minimum.nil?
+
+        below = counts.select { |locale_counts| locale_counts.percent < minimum }
+        return if below.empty?
+
+        raise Errors::CoverageError,
+              "Translation coverage is below translation.min_coverage (#{minimum}%): #{below.join('; ')}"
+      end
+
+      private
+
+      def count(locale, entries)
+        tally = Hash.new(0)
+        entries.each do |msgid, entry|
+          next unless @msgids.include?(msgid)
+
+          entry_state = state(msgid, entry)
+          tally[entry_state] += 1
+          @invalid[locale] << msgid if entry_state == :invalid
+        end
+        Counts.new(locale, tally[:translated], tally[:fuzzy], tally[:untranslated], tally[:invalid])
+      end
+
+      def state(msgid, entry)
+        if entry[:fuzzy] then :fuzzy
+        elsif entry[:msgstr].to_s.empty? then :untranslated
+        elsif MsgstrValidator.valid?(msgid, entry[:msgstr]) then :translated
+        else :invalid
+        end
+      end
+    end
+  end
+end

data/lib/jekyll-l10n/extraction/dom_text_extractor.rb

@@ -3,8 +3,6 @@
 require_relative '../utils/text_normalizer'
 require_relative '../utils/xpath_reference_generator'
 require_relative '../utils/html_text_utils'
-require_relative '../utils/html_elements'
-require_relative '../utils/text_validator'
 
 module Jekyll
   module L10n
@@ -27,7 +25,7 @@ module Jekyll
     #   entry = DomTextExtractor.extract(node, 'docs/index.html', '_site')
     #   # Returns hash with :msgid, :msgstr, :reference if valid text found
     module DomTextExtractor
-      extend self
+      module_function
 
       # Extract text content from an HTML element.
       #
@@ -56,34 +54,7 @@ module Jekyll
       end
 
       def extract_block_text(node)
-        return nil if only_contains_block_elements?(node)
-        return nil if HtmlTextUtils.layout_block_children?(node)
-
-        text = HtmlTextUtils.extract_with_inline_tags(node)
-        TextValidator.valid?(text) ? text : nil
-      end
-
-      def only_contains_block_elements?(node)
-        node.children.each do |child|
-          return false if non_empty_text?(child)
-          return false if non_block_element?(child)
-        end
-
-        block_element_children?(node)
-      end
-
-      private
-
-      def non_empty_text?(child)
-        child.text? && !child.content.strip.empty?
-      end
-
-      def non_block_element?(child)
-        child.element? && !HtmlTextUtils::ALL_BLOCK_ELEMENTS.include?(child.name)
-      end
-
-      def block_element_children?(node)
-        node.children.any? { |c| c.element? && HtmlTextUtils::ALL_BLOCK_ELEMENTS.include?(c.name) }
+        HtmlTextUtils.block_text(node)
       end
     end
   end