jekyll-l10n 1.7.1 → 3.0.0

data/lib/jekyll-l10n/extraction/extractor.rb

@@ -1,83 +1,71 @@
 # frozen_string_literal: true
 
+require 'set'
+require_relative '../configuration'
+require_relative '../constants'
 require_relative 'config_loader'
 require_relative 'result_saver'
 require_relative 'logger'
 require_relative 'html_string_extractor'
+require_relative 'coverage_report'
 require_relative '../utils/file_operations'
 require_relative '../utils/site_config_accessor'
 require_relative '../utils/error_handler'
 
 module Jekyll
   module L10n
-    # String Extraction Orchestrator - Finds translatable strings in generated HTML
+    # Collects translatable strings from the built site into PO files.
     #
-    # The Extractor is the main entry point for the string extraction workflow. It scans all
-    # generated HTML files after Jekyll's build, identifies translatable content (text nodes
-    # and configurable HTML attributes), and creates or updates GNU Gettext PO files with
-    # the extracted strings.
+    # PostWriteProcessor runs it after Jekyll writes the site. Each build it:
+    # 1. Reads every English HTML file in the destination. It skips locale
+    #    folders such as `es/`.
+    # 2. Writes one PO file per page with the page's text and translatable
+    #    attributes.
+    # 3. Merges those files into one compendium per locale (unless
+    #    `update_compendium` is false). The merge deletes the page files.
+    # 4. Machine-translates the compendia when LibreTranslate is on.
+    # 5. Prints a summary and each locale's translation coverage unless
+    #    `logging.show_statistics` is false.
+    # 6. Fails the build when a locale's coverage is below
+    #    `translation.min_coverage`.
     #
-    # The extraction workflow:
-    # 1. Scans all HTML files in Jekyll output directory (_site/)
-    # 2. For each HTML file, extracts translatable text and attributes
-    # 3. Normalizes text for consistent matching across builds
-    # 4. Creates or updates page-specific PO files in _locales/ directory
-    # 5. Optionally applies automatic translations via LibreTranslate API
+    # It always processes the whole site. Incremental mode does not affect it.
     #
-    # Key responsibilities:
-    # - Load and validate extraction configuration from pages
-    # - Extract text and attributes from HTML with file location references
-    # - Create and update PO files with extracted strings
-    # - Log extraction statistics and progress
-    # - Coordinate with LibreTranslate for automatic translation
-    #
-    # @example Usage (typically invoked by PostWriteProcessor)
-    #   extractor = Extractor.new(site)
-    #   result = extractor.extract_site
+    # @example
+    #   Extractor.new(site).extract_site
     #   # => { files_processed: 42, strings_extracted: 237, po_files_created: 3 }
     #
-    # @see Jekyll::L10n::ExtractionResultSaver for PO file creation and updates
-    # @see Jekyll::L10n::HtmlStringExtractor for text and attribute extraction
+    # @see ExtractionResultSaver writes and merges the PO files
+    # @see HtmlStringExtractor finds strings in one HTML file
     #
     class Extractor
       # @!attribute [r] site
-      #   The Jekyll site object with generated pages
       #   @return [Jekyll::Site]
       attr_reader :site
 
-      # Initialize the string extractor
-      #
-      # Sets up configuration and result saving infrastructure for extraction.
-      #
-      # @param site [Jekyll::Site] The Jekyll site object
+      # @param site [Jekyll::Site] A site that Jekyll has already written
       def initialize(site)
         @site = site
         @source = SiteConfigAccessor.source(@site)
         @dest = SiteConfigAccessor.dest(@site)
         @config_loader = ExtractionConfigLoader.new(@site, @dest)
         @result_saver = ExtractionResultSaver.new(@site)
+        @extracted_msgids = Set.new
       end
 
-      # Extract all translatable strings from the generated site
+      # Runs all the steps in the class description.
       #
-      # Main entry point for extraction. Scans all HTML files in the build output,
-      # extracts translatable strings and attributes, creates/updates PO files,
-      # and optionally translates strings via LibreTranslate API.
-      #
-      # @return [Hash<Symbol, Integer>] Statistics hash with keys:
-      #   - :files_processed - Number of HTML files processed
-      #   - :strings_extracted - Total strings extracted
-      #   - :po_files_created - Number of PO files created/updated
-      # @example
-      #   result = extractor.extract_site
-      #   puts "Processed #{result[:files_processed]} files"
+      # @return [Hash<Symbol, Integer>] Counts under :files_processed,
+      #   :strings_extracted, and :po_files_created (files created or updated)
+      # @raise [Errors::CoverageError] When a locale is below `translation.min_coverage`
       def extract_site
-        Jekyll.logger.info 'Localization', 'Extracting translatable strings...'
+        Jekyll.logger.info Constants::LOG_PREFIX, 'Extracting translatable strings...'
         start_time = Time.now
         stats = process_all_html_files
         @result_saver.finalize_compendia
         translate_all_compendia
-        ExtractionLogger.log_summary(stats, Time.now - start_time)
+        ExtractionLogger.log_summary(stats, Time.now - start_time) if show_statistics?
+        check_coverage
         stats
       end
 
@@ -106,20 +94,6 @@ module Jekyll
         end
       end
 
-      def process_file(file_path)
-        return default_stats unless @config_loader.valid_for_extraction?(file_path)
-
-        config = @config_loader.load_page_config(file_path)
-        entries = extract_strings_from_file(file_path, config)
-        return default_stats if entries.empty?
-
-        page_path = construct_page_path(file_path)
-        @result_saver.save_results(config, entries, page_path)
-      rescue StandardError => e
-        ExtractionLogger.log_error(file_path, e)
-        default_stats
-      end
-
       def default_stats
         { files_processed: 0, strings_extracted: 0, po_files_created: 0 }
       end
@@ -130,24 +104,57 @@ module Jekyll
         html = FileOperations.read_utf8(file_path)
         exclude_selectors = @config_loader.extract_exclude_selectors(config)
 
-        extractor = HtmlStringExtractor.new(config.translatable_attributes, exclude_selectors)
+        extractor = HtmlStringExtractor.new(config.translatable_attributes, exclude_selectors,
+                                            baseurl: @site.config['baseurl'] || '')
         extractor.extract(html, @dest, file_path)
       end
 
+      # Machine translation is a site-wide step: it reads the `jekyll_l10n`
+      # settings and covers every locale the site uses.
+      #
+      # @return [PageLocalesConfig, nil] The site config, or nil when
+      #   LibreTranslate is off
       def find_libretranslate_config
         return nil unless @site.respond_to?(:pages)
 
-        @site.pages.each do |page|
-          next unless page.data['with_locales'] == true
+        config = SiteConfigAccessor.site_config(@site, all_locales: true)
+        config if config.libretranslate_enabled?
+      end
 
-          config = @config_loader.load_page_config(page.destination(''))
-          return config if config.libretranslate_enabled?
-        end
+      private
+
+      # per-page body called from the html_files loop in process_all_html_files
+      def process_file(file_path)
+        return default_stats unless @config_loader.valid_for_extraction?(file_path)
+
+        config = @config_loader.load_page_config(file_path)
+        entries = extract_strings_from_file(file_path, config)
+        return default_stats if entries.empty?
+
+        @extracted_msgids.merge(entries.map { |entry| entry[:msgid] })
 
-        nil
+        page_path = construct_page_path(file_path)
+        @result_saver.save_results(config, entries, page_path)
+      rescue StandardError => e
+        ExtractionLogger.log_error(file_path, e)
+        default_stats
       end
 
-      private
+      # Runs after machine translation, so its entries count, and outside
+      # ErrorHandler, so a CoverageError fails the build.
+      def check_coverage
+        config = SiteConfigAccessor.site_config(@site, all_locales: true)
+        report = CoverageReport.new(@site, config, @extracted_msgids)
+        report.warn_invalid
+        report.log if config.show_statistics?
+        report.enforce!(config.min_coverage)
+      end
+
+      # The summary covers the whole site, so read the site-wide setting, not a
+      # page's.
+      def show_statistics?
+        SiteConfigAccessor.site_config(@site).show_statistics?
+      end
 
       def construct_page_path(file_path)
         file_path.sub("#{@dest}/", '')

data/lib/jekyll-l10n/extraction/html_string_extractor.rb

@@ -1,8 +1,12 @@
 # frozen_string_literal: true
 
+require_relative '../constants'
 require_relative 'dom_text_extractor'
 require_relative 'dom_attribute_extractor'
 require_relative 'logger'
+require_relative '../utils/baseurl_normalizer'
+require_relative '../utils/html_elements'
+require_relative '../utils/html_parser'
 
 module Jekyll
   module L10n
@@ -27,7 +31,7 @@ module Jekyll
     #   entries = extractor.extract(html_content, '_site', 'docs/index.html')
     #   # Returns array of hash entries with :msgid, :msgstr, :reference keys
     class HtmlStringExtractor
-      attr_reader :translatable_attrs, :exclude_selectors
+      attr_reader :translatable_attrs, :exclude_selectors, :baseurl
 
       # Initialize a new HtmlStringExtractor.
       #
@@ -35,9 +39,12 @@ module Jekyll
       #   (e.g., ['title', 'alt', 'aria-label', 'placeholder', 'aria-description'])
       # @param exclude_selectors [Array<String>] CSS selectors for elements to skip
       #   during extraction (e.g., ['script', 'style', '.no-translate'])
-      def initialize(translatable_attrs, exclude_selectors)
+      # @param baseurl [String] (keyword) Site baseurl, kept out of msgids
+      #   (see {BaseurlNormalizer})
+      def initialize(translatable_attrs, exclude_selectors, baseurl: '')
         @translatable_attrs = translatable_attrs
         @exclude_selectors = exclude_selectors
+        @baseurl = baseurl
       end
 
       # Extract translatable strings from HTML.
@@ -57,7 +64,8 @@ module Jekyll
       def extract(html, dest, file_path)
         entries = []
 
-        doc = Nokogiri::HTML(html)
+        doc = HtmlParser.parse_document(html)
+        BaseurlNormalizer.strip(doc, @baseurl)
         walk_dom(doc.root, file_path, entries, dest)
 
         entries.uniq { |e| e[:msgid] }
@@ -65,17 +73,23 @@ module Jekyll
 
       private
 
-      def walk_dom(node, file_path, entries, dest)
+      # Below a content element that has a msgid, inline text is translated only
+      # as part of that msgid, so it gets none of its own (covered). A nested
+      # content element is looked up by its own msgid, so it is extracted again
+      # and decides coverage for its own descendants.
+      def walk_dom(node, file_path, entries, dest, covered: false)
         return if node.nil? || should_exclude_element?(node)
 
-        process_text_content(node, file_path, entries, dest)
+        block = node.element? && HtmlElements::CONTENT_ELEMENTS.include?(node.name)
+        entry = process_text_content(node, file_path, entries, dest) if block || !covered
         process_attributes(node, file_path, entries, dest)
-        process_children(node, file_path, entries, dest)
+        process_children(node, file_path, entries, dest, covered: block ? !entry.nil? : covered)
       end
 
       def process_text_content(node, file_path, entries, dest)
         entry = DomTextExtractor.extract(node, file_path, dest)
         entries << entry if entry
+        entry
       end
 
       def process_attributes(node, file_path, entries, dest)
@@ -83,8 +97,8 @@ module Jekyll
         entries.concat(attr_entries)
       end
 
-      def process_children(node, file_path, entries, dest)
-        node.children.each { |child| walk_dom(child, file_path, entries, dest) }
+      def process_children(node, file_path, entries, dest, covered: false)
+        node.children.each { |child| walk_dom(child, file_path, entries, dest, covered: covered) }
       end
 
       def should_exclude_element?(node)
@@ -92,10 +106,10 @@ module Jekyll
 
         @exclude_selectors.any? { |selector| node.matches?(selector) }
       rescue Nokogiri::CSS::SyntaxError => e
-        Jekyll.logger.warn 'Localization', "CSS selector syntax error: #{e.message}"
+        Jekyll.logger.warn Constants::LOG_PREFIX, "CSS selector syntax error: #{e.message}"
         false
       rescue StandardError => e
-        Jekyll.logger.warn 'Localization', "Selector matching error (continuing): #{e.message}"
+        Jekyll.logger.warn Constants::LOG_PREFIX, "Selector matching error (continuing): #{e.message}"
         false
       end
     end

data/lib/jekyll-l10n/extraction/logger.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative '../constants'
 require_relative '../utils/error_handler'
 
 module Jekyll
@@ -39,11 +40,11 @@ module Jekyll
       # @param elapsed [Float] Time elapsed in seconds
       # @return [void]
       def self.log_summary(stats, elapsed)
-        Jekyll.logger.info 'Localization', 'Extraction complete:'
-        Jekyll.logger.info 'Localization', "  Files processed: #{stats[:files_processed]}"
-        Jekyll.logger.info 'Localization', "  Strings extracted: #{stats[:strings_extracted]}"
-        Jekyll.logger.info 'Localization', "  PO files created/updated: #{stats[:po_files_created]}"
-        Jekyll.logger.info 'Localization', "  Time: #{elapsed.round(2)}s"
+        Jekyll.logger.info Constants::LOG_PREFIX, 'Extraction complete:'
+        Jekyll.logger.info Constants::LOG_PREFIX, "  Files processed: #{stats[:files_processed]}"
+        Jekyll.logger.info Constants::LOG_PREFIX, "  Strings extracted: #{stats[:strings_extracted]}"
+        Jekyll.logger.info Constants::LOG_PREFIX, "  PO files created/updated: #{stats[:po_files_created]}"
+        Jekyll.logger.info Constants::LOG_PREFIX, "  Time: #{elapsed.round(2)}s"
       end
     end
   end

data/lib/jekyll-l10n/extraction/result_saver.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative '../configuration'
 require_relative '../po_file/manager'
 require_relative '../utils/page_locales_config'
 require_relative '../utils/site_config_accessor'
@@ -33,8 +34,7 @@ module Jekyll
       # @param site [Jekyll::Site] Jekyll site object
       def initialize(site)
         @site = site
-        with_locales_data = SiteConfigAccessor.extract_locales_data(@site)
-        @site_config = PageLocalesConfig.new({ 'with_locales_data' => with_locales_data })
+        @site_config = SiteConfigAccessor.site_config(@site)
       end
 
       # Save extraction results to PO files.
@@ -105,17 +105,12 @@ module Jekyll
         po_files_created
       end
 
-      # Build a PageLocalesConfig whose locale list is the union of site-wide
-      # defaults and every page-level front-matter locale override.
-      #
-      # All other site-wide settings (locales_dir, extraction, translation,
-      # logging, etc.) are preserved unchanged; only the locale list is widened.
+      # The site config with every locale any page uses, not only the
+      # site-wide list (see SiteConfigAccessor.all_configured_locales).
       #
       # @return [PageLocalesConfig]
       def merge_config
-        all_locales = SiteConfigAccessor.all_configured_locales(@site)
-        merged_data = (@site_config.data['with_locales_data'] || {}).merge('locales' => all_locales)
-        PageLocalesConfig.new({ 'with_locales_data' => merged_data })
+        SiteConfigAccessor.site_config(@site, all_locales: true)
       end
 
       private :save_po_files