jekyll-l10n 2.0.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +44 -0
  3. data/lib/jekyll-l10n/configuration/schema.rb +308 -0
  4. data/lib/jekyll-l10n/configuration.rb +165 -0
  5. data/lib/jekyll-l10n/constants.rb +45 -10
  6. data/lib/jekyll-l10n/errors.rb +7 -0
  7. data/lib/jekyll-l10n/extraction/compendium_merger.rb +9 -11
  8. data/lib/jekyll-l10n/extraction/compendium_translator.rb +27 -11
  9. data/lib/jekyll-l10n/extraction/config_loader.rb +4 -4
  10. data/lib/jekyll-l10n/extraction/coverage_report.rb +129 -0
  11. data/lib/jekyll-l10n/extraction/dom_text_extractor.rb +2 -31
  12. data/lib/jekyll-l10n/extraction/extractor.rb +48 -28
  13. data/lib/jekyll-l10n/extraction/html_string_extractor.rb +24 -10
  14. data/lib/jekyll-l10n/extraction/logger.rb +6 -5
  15. data/lib/jekyll-l10n/extraction/result_saver.rb +5 -10
  16. data/lib/jekyll-l10n/instrumentation.rb +172 -139
  17. data/lib/jekyll-l10n/jekyll/generator.rb +10 -22
  18. data/lib/jekyll-l10n/jekyll/localized_page.rb +27 -49
  19. data/lib/jekyll-l10n/jekyll/post_write_html_reprocessor.rb +42 -25
  20. data/lib/jekyll-l10n/jekyll/post_write_processor.rb +2 -1
  21. data/lib/jekyll-l10n/jekyll/regeneration_checker.rb +16 -10
  22. data/lib/jekyll-l10n/jekyll/url_filter.rb +8 -7
  23. data/lib/jekyll-l10n/po_file/entry_parser.rb +155 -0
  24. data/lib/jekyll-l10n/po_file/fuzzy_matcher.rb +67 -53
  25. data/lib/jekyll-l10n/po_file/loader.rb +20 -11
  26. data/lib/jekyll-l10n/po_file/manager.rb +32 -17
  27. data/lib/jekyll-l10n/po_file/merger.rb +2 -1
  28. data/lib/jekyll-l10n/po_file/path_builder.rb +11 -1
  29. data/lib/jekyll-l10n/po_file/reader.rb +40 -508
  30. data/lib/jekyll-l10n/po_file/writer.rb +31 -16
  31. data/lib/jekyll-l10n/translation/block_text_extractor.rb +3 -7
  32. data/lib/jekyll-l10n/translation/html_translator.rb +136 -157
  33. data/lib/jekyll-l10n/translation/libre_translate_languages.rb +73 -0
  34. data/lib/jekyll-l10n/translation/libre_translator.rb +134 -78
  35. data/lib/jekyll-l10n/translation/request_slots.rb +69 -0
  36. data/lib/jekyll-l10n/translation/translation_progress.rb +77 -0
  37. data/lib/jekyll-l10n/translation/translator.rb +20 -17
  38. data/lib/jekyll-l10n/utils/baseurl_normalizer.rb +108 -0
  39. data/lib/jekyll-l10n/utils/debug_logger.rb +4 -51
  40. data/lib/jekyll-l10n/utils/error_handler.rb +9 -24
  41. data/lib/jekyll-l10n/utils/external_link_icon_preserver.rb +5 -62
  42. data/lib/jekyll-l10n/utils/html_parser.rb +24 -31
  43. data/lib/jekyll-l10n/utils/html_text_utils.rb +146 -91
  44. data/lib/jekyll-l10n/utils/logger_formatter.rb +6 -4
  45. data/lib/jekyll-l10n/utils/msgstr_validator.rb +74 -0
  46. data/lib/jekyll-l10n/utils/page_locales_config.rb +126 -107
  47. data/lib/jekyll-l10n/utils/po_entry_converter.rb +0 -43
  48. data/lib/jekyll-l10n/utils/site_config_accessor.rb +56 -54
  49. data/lib/jekyll-l10n/utils/translation_resolver.rb +56 -6
  50. data/lib/jekyll-l10n/utils/url_path_builder.rb +5 -27
  51. data/lib/jekyll-l10n/utils/url_transformer.rb +5 -11
  52. data/lib/jekyll-l10n/version.rb +1 -1
  53. data/lib/jekyll-l10n.rb +9 -6
  54. metadata +12 -3
@@ -4,7 +4,6 @@ require 'fileutils'
4
4
  require_relative '../po_file/manager'
5
5
  require_relative '../po_file/fuzzy_matcher'
6
6
  require_relative '../utils/page_locales_config'
7
- require_relative '../utils/site_config_accessor'
8
7
 
9
8
  module Jekyll
10
9
  module L10n
@@ -35,8 +34,6 @@ module Jekyll
35
34
  # @param site [Jekyll::Site] Jekyll site object
36
35
  def initialize(site)
37
36
  @site = site
38
- with_locales_data = SiteConfigAccessor.extract_locales_data(@site)
39
- @site_config = PageLocalesConfig.new({ 'with_locales_data' => with_locales_data })
40
37
  end
41
38
 
42
39
  # Merge page-specific PO files into compendia for all locales.
@@ -49,8 +46,11 @@ module Jekyll
49
46
  # @param config [PageLocalesConfig] Localization configuration with locales list
50
47
  # @return [void]
51
48
  def merge_compendia(po_manager, config)
49
+ # Fuzzy scores depend only on the two English strings, and every
50
+ # locale compares the same new strings, so they are scored once per run.
51
+ @fuzzy_scores = {}
52
52
  config.locales.each do |locale|
53
- process_locale(locale, po_manager, config)
53
+ process_locale(locale, po_manager)
54
54
  end
55
55
  end
56
56
 
@@ -58,9 +58,8 @@ module Jekyll
58
58
 
59
59
  # Process a single locale: merge existing compendium with new page translations
60
60
  # Skips writing when translations are unchanged to avoid unnecessary file rewrites
61
- def process_locale(locale, po_manager, config)
62
- compendium_path = File.join(@site.source, config.locales_dir, "#{locale}.po")
63
- existing_compendium = load_existing_compendium(compendium_path)
61
+ def process_locale(locale, po_manager)
62
+ existing_compendium = load_existing_compendium(po_manager.compendium_path(locale))
64
63
  merged = po_manager.merge_po_files(locale)
65
64
 
66
65
  combined = build_combined_hash(existing_compendium)
@@ -70,7 +69,7 @@ module Jekyll
70
69
  combined_entries = format_compendium_entries(combined)
71
70
  po_manager.save_compendium(locale, combined_entries)
72
71
  end
73
- cleanup_locale_directory(locale, config)
72
+ cleanup_locale_directory(po_manager.locale_dir(locale))
74
73
  end
75
74
 
76
75
  # Check if compendium content is unchanged (same msgids and msgstrs)
@@ -146,7 +145,7 @@ module Jekyll
146
145
  end
147
146
 
148
147
  def resolve_new_entry(combined, msgid, entry, fuzzy_candidates, matched_old_msgids)
149
- match = PoFuzzyMatcher.find_match(msgid, fuzzy_candidates)
148
+ match = PoFuzzyMatcher.find_match(msgid, fuzzy_candidates, scores: @fuzzy_scores)
150
149
  if match
151
150
  combined[msgid] = create_fuzzy_entry(entry, match)
152
151
  matched_old_msgids << match[:msgid]
@@ -195,8 +194,7 @@ module Jekyll
195
194
 
196
195
  # IMPORTANT: Remove locale-specific directory after compendium merge
197
196
  # This cleans up page-specific PO files that have been merged into the compendium
198
- def cleanup_locale_directory(locale, config)
199
- locale_dir = File.join(@site.source, config.locales_dir, locale)
197
+ def cleanup_locale_directory(locale_dir)
200
198
  FileUtils.rm_rf(locale_dir) if File.directory?(locale_dir)
201
199
  end
202
200
 
@@ -1,9 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative '../constants'
4
+ require_relative '../instrumentation'
3
5
  require_relative '../po_file/manager'
4
6
  require_relative '../utils/page_locales_config'
5
7
  require_relative '../utils/po_entry_converter'
6
- require_relative '../utils/site_config_accessor'
7
8
  require_relative '../utils/logger_formatter'
8
9
  require_relative '../translation/libre_translator'
9
10
 
@@ -33,18 +34,19 @@ module Jekyll
33
34
  # @param site [Jekyll::Site] Jekyll site object
34
35
  def initialize(site)
35
36
  @site = site
36
- with_locales_data = SiteConfigAccessor.extract_locales_data(@site)
37
- @site_config = PageLocalesConfig.new({ 'with_locales_data' => with_locales_data })
38
37
  end
39
38
 
40
39
  # Translate compendia for all configured locales.
41
40
  #
42
41
  # Checks if LibreTranslate is enabled. If so, for each configured locale,
43
42
  # loads the compendium PO file, identifies untranslated entries, sends them
44
- # to LibreTranslate API, and saves the updated file.
43
+ # to LibreTranslate API, and saves the updated file. Locales are
44
+ # translated at once; their requests share libretranslate_concurrency.
45
45
  #
46
46
  # @param config [PageLocalesConfig] Localization configuration
47
47
  # @return [void]
48
+ # @raise [LibreTranslator::TranslationError] If a locale failed, once
49
+ # every locale has finished; the locales that finished are saved
48
50
  def translate_compendia(config)
49
51
  po_manager = PoFileManager.new(@site, config.locales_dir)
50
52
  translate_compendia_for_locale(po_manager, config)
@@ -59,15 +61,29 @@ module Jekyll
59
61
  log_translation_start(config)
60
62
  translator = LibreTranslator.new(config)
61
63
 
62
- config.locales.each do |locale|
63
- process_single_locale(locale, config, translator, po_manager)
64
+ threads = config.locales.map do |locale|
65
+ Instrumentation.traced_thread do
66
+ Thread.current.report_on_exception = false
67
+ process_single_locale(locale, translator, po_manager)
68
+ end
64
69
  end
70
+ join_locales(threads)
65
71
 
66
72
  log_translation_complete(config)
67
73
  end
68
74
 
69
- def process_single_locale(locale, config, translator, po_manager)
70
- compendium_path = File.join(@site.source, config.locales_dir, "#{locale}.po")
75
+ def join_locales(threads)
76
+ first_error = nil
77
+ threads.each do |thread|
78
+ thread.join
79
+ rescue StandardError => e
80
+ first_error ||= e
81
+ end
82
+ raise first_error if first_error
83
+ end
84
+
85
+ def process_single_locale(locale, translator, po_manager)
86
+ compendium_path = po_manager.compendium_path(locale)
71
87
  LoggerFormatter.debug_if_enabled('CompendiumTranslator',
72
88
  "Processing compendium file: #{compendium_path}")
73
89
  return unless File.exist?(compendium_path)
@@ -119,17 +135,17 @@ module Jekyll
119
135
 
120
136
  def log_translation_start(config)
121
137
  locales = config.locales.join(', ')
122
- Jekyll.logger.info 'Localization',
138
+ Jekyll.logger.info Constants::LOG_PREFIX,
123
139
  "Starting LibreTranslate translation for locales: #{locales}"
124
140
  end
125
141
 
126
142
  def log_translation_complete(config)
127
143
  locales = config.locales.join(', ')
128
- Jekyll.logger.info 'Localization',
144
+ Jekyll.logger.info Constants::LOG_PREFIX,
129
145
  "LibreTranslate translation complete for locales: #{locales}"
130
146
  end
131
147
 
132
- private :translate_compendia_for_locale, :process_single_locale,
148
+ private :translate_compendia_for_locale, :join_locales, :process_single_locale,
133
149
  :log_compendia_enabled_check, :log_compendium_stats,
134
150
  :log_translation_complete_for_locale, :log_compendium_saved,
135
151
  :count_empty_entries, :log_translation_start, :log_translation_complete
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative '../configuration'
3
4
  require_relative '../utils/page_locales_config'
4
5
  require_relative '../utils/locale_url_formatter'
5
6
  require_relative '../utils/site_config_accessor'
@@ -51,7 +52,7 @@ module Jekyll
51
52
  # @return [Hash, nil] The page data, or nil when no localized page matches
52
53
  def find_page_config_for_file(file_path)
53
54
  @site.pages.each do |page|
54
- next unless page.data['with_locales'] == true
55
+ next unless Configuration.localizable?(page)
55
56
 
56
57
  page_output = page.output_ext ? page.destination('') : page.destination('/')
57
58
  next unless file_path.end_with?(page_output.sub(%r{/$}, '/index.html'))
@@ -77,10 +78,9 @@ module Jekyll
77
78
 
78
79
  # @param config [PageLocalesConfig] The page's settings
79
80
  # @return [Array<String>] CSS selectors of elements to leave out of
80
- # extraction, from `extraction.exclude_selectors` or the defaults below
81
+ # extraction (see PageLocalesConfig#exclude_selectors)
81
82
  def extract_exclude_selectors(config)
82
- config.data.dig('with_locales_data', 'extraction', 'exclude_selectors') ||
83
- ['script', 'style', 'code.language-plaintext', 'pre code']
83
+ config.exclude_selectors
84
84
  end
85
85
  end
86
86
  end
@@ -0,0 +1,129 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../constants'
4
+ require_relative '../errors'
5
+ require_relative '../po_file/path_builder'
6
+ require_relative '../po_file/reader'
7
+ require_relative '../utils/msgstr_validator'
8
+
9
+ module Jekyll
10
+ module L10n
11
+ # Counts how much of each locale's compendium is translated.
12
+ #
13
+ # Each entry counts once, as translated, fuzzy, untranslated, or invalid
14
+ # (a msgstr whose markup MsgstrValidator rejects, so pages use the
15
+ # fallback). Extractor warns about each invalid entry, prints the counts
16
+ # with the extraction summary, and fails the build when a locale is below
17
+ # `translation.min_coverage`.
18
+ #
19
+ # Only entries whose msgid this build extracted count: a compendium also
20
+ # keeps entries for text the site no longer has. The counts come from the
21
+ # compendium files as they are after extraction, so with
22
+ # `update_compendium: false` they leave out strings no compendium holds
23
+ # yet.
24
+ #
25
+ # @example
26
+ # report = CoverageReport.new(site, SiteConfigAccessor.site_config(site, all_locales: true),
27
+ # extracted_msgids)
28
+ # report.warn_invalid
29
+ # report.log
30
+ # report.enforce!(90) # raises Errors::CoverageError below 90%
31
+ class CoverageReport
32
+ # Entry counts for one locale.
33
+ Counts = Struct.new(:locale, :translated, :fuzzy, :untranslated, :invalid) do
34
+ # @return [Integer] Entries in the compendium
35
+ def total
36
+ translated + fuzzy + untranslated + invalid
37
+ end
38
+
39
+ # @return [Float] Translated entries as a percentage; 100 when the
40
+ # compendium is empty
41
+ def percent
42
+ total.zero? ? 100.0 : translated * 100.0 / total
43
+ end
44
+
45
+ # @return [String] e.g. "fr: 92.5% translated (1200 of 1297; 40 fuzzy, 50 untranslated, 7 invalid)"
46
+ def to_s
47
+ "#{locale}: #{percent.round(1)}% translated (#{translated} of #{total}; " \
48
+ "#{fuzzy} fuzzy, #{untranslated} untranslated, #{invalid} invalid)"
49
+ end
50
+ end
51
+
52
+ # @param site [Jekyll::Site]
53
+ # @param config [PageLocalesConfig] The site config, for locales and locales_dir
54
+ # @param msgids [Set<String>] The msgids this build extracted
55
+ def initialize(site, config, msgids)
56
+ @site = site
57
+ @config = config
58
+ @msgids = msgids
59
+ @invalid = Hash.new { |hash, locale| hash[locale] = [] }
60
+ end
61
+
62
+ # @return [Array<Counts>] One entry per locale that has a compendium
63
+ def counts
64
+ @counts ||= @config.locales.filter_map do |locale|
65
+ path = PoPathBuilder.build(@site.source, @config.locales_dir, locale, nil)
66
+ count(locale, PoFileReader.parse_for_merge(path)) if File.exist?(path)
67
+ end
68
+ end
69
+
70
+ # Warn about each entry whose msgstr markup does not match its msgid.
71
+ #
72
+ # @return [void]
73
+ def warn_invalid
74
+ counts.each do |locale_counts|
75
+ @invalid[locale_counts.locale].each do |msgid|
76
+ Jekyll.logger.warn Constants::LOG_PREFIX,
77
+ "#{locale_counts.locale}: msgstr markup does not match its msgid, " \
78
+ "so pages show the fallback: #{msgid[0..Constants::LOG_TRUNCATE_LONG]}"
79
+ end
80
+ end
81
+ end
82
+
83
+ # Print one line per locale.
84
+ #
85
+ # @return [void]
86
+ def log
87
+ return if counts.empty?
88
+
89
+ Jekyll.logger.info Constants::LOG_PREFIX, 'Translation coverage:'
90
+ counts.each { |locale_counts| Jekyll.logger.info Constants::LOG_PREFIX, " #{locale_counts}" }
91
+ end
92
+
93
+ # @param minimum [Integer, nil] Required percentage, or nil for none
94
+ # @return [void]
95
+ # @raise [Errors::CoverageError] When any locale is below the minimum
96
+ def enforce!(minimum)
97
+ return if minimum.nil?
98
+
99
+ below = counts.select { |locale_counts| locale_counts.percent < minimum }
100
+ return if below.empty?
101
+
102
+ raise Errors::CoverageError,
103
+ "Translation coverage is below translation.min_coverage (#{minimum}%): #{below.join('; ')}"
104
+ end
105
+
106
+ private
107
+
108
+ def count(locale, entries)
109
+ tally = Hash.new(0)
110
+ entries.each do |msgid, entry|
111
+ next unless @msgids.include?(msgid)
112
+
113
+ entry_state = state(msgid, entry)
114
+ tally[entry_state] += 1
115
+ @invalid[locale] << msgid if entry_state == :invalid
116
+ end
117
+ Counts.new(locale, tally[:translated], tally[:fuzzy], tally[:untranslated], tally[:invalid])
118
+ end
119
+
120
+ def state(msgid, entry)
121
+ if entry[:fuzzy] then :fuzzy
122
+ elsif entry[:msgstr].to_s.empty? then :untranslated
123
+ elsif MsgstrValidator.valid?(msgid, entry[:msgstr]) then :translated
124
+ else :invalid
125
+ end
126
+ end
127
+ end
128
+ end
129
+ end
@@ -3,8 +3,6 @@
3
3
  require_relative '../utils/text_normalizer'
4
4
  require_relative '../utils/xpath_reference_generator'
5
5
  require_relative '../utils/html_text_utils'
6
- require_relative '../utils/html_elements'
7
- require_relative '../utils/text_validator'
8
6
 
9
7
  module Jekyll
10
8
  module L10n
@@ -27,7 +25,7 @@ module Jekyll
27
25
  # entry = DomTextExtractor.extract(node, 'docs/index.html', '_site')
28
26
  # # Returns hash with :msgid, :msgstr, :reference if valid text found
29
27
  module DomTextExtractor
30
- extend self
28
+ module_function
31
29
 
32
30
  # Extract text content from an HTML element.
33
31
  #
@@ -56,34 +54,7 @@ module Jekyll
56
54
  end
57
55
 
58
56
  def extract_block_text(node)
59
- return nil if only_contains_block_elements?(node)
60
- return nil if HtmlTextUtils.layout_block_children?(node)
61
-
62
- text = HtmlTextUtils.extract_with_inline_tags(node)
63
- TextValidator.valid?(text) ? text : nil
64
- end
65
-
66
- def only_contains_block_elements?(node)
67
- node.children.each do |child|
68
- return false if non_empty_text?(child)
69
- return false if non_block_element?(child)
70
- end
71
-
72
- block_element_children?(node)
73
- end
74
-
75
- private
76
-
77
- def non_empty_text?(child)
78
- child.text? && !child.content.strip.empty?
79
- end
80
-
81
- def non_block_element?(child)
82
- child.element? && !HtmlTextUtils::ALL_BLOCK_ELEMENTS.include?(child.name)
83
- end
84
-
85
- def block_element_children?(node)
86
- node.children.any? { |c| c.element? && HtmlTextUtils::ALL_BLOCK_ELEMENTS.include?(c.name) }
57
+ HtmlTextUtils.block_text(node)
87
58
  end
88
59
  end
89
60
  end
@@ -1,9 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require 'set'
4
+ require_relative '../configuration'
5
+ require_relative '../constants'
3
6
  require_relative 'config_loader'
4
7
  require_relative 'result_saver'
5
8
  require_relative 'logger'
6
9
  require_relative 'html_string_extractor'
10
+ require_relative 'coverage_report'
7
11
  require_relative '../utils/file_operations'
8
12
  require_relative '../utils/site_config_accessor'
9
13
  require_relative '../utils/error_handler'
@@ -20,7 +24,10 @@ module Jekyll
20
24
  # 3. Merges those files into one compendium per locale (unless
21
25
  # `update_compendium` is false). The merge deletes the page files.
22
26
  # 4. Machine-translates the compendia when LibreTranslate is on.
23
- # 5. Prints a summary unless `logging.show_statistics` is false.
27
+ # 5. Prints a summary and each locale's translation coverage unless
28
+ # `logging.show_statistics` is false.
29
+ # 6. Fails the build when a locale's coverage is below
30
+ # `translation.min_coverage`.
24
31
  #
25
32
  # It always processes the whole site. Incremental mode does not affect it.
26
33
  #
@@ -43,19 +50,22 @@ module Jekyll
43
50
  @dest = SiteConfigAccessor.dest(@site)
44
51
  @config_loader = ExtractionConfigLoader.new(@site, @dest)
45
52
  @result_saver = ExtractionResultSaver.new(@site)
53
+ @extracted_msgids = Set.new
46
54
  end
47
55
 
48
56
  # Runs all the steps in the class description.
49
57
  #
50
58
  # @return [Hash<Symbol, Integer>] Counts under :files_processed,
51
59
  # :strings_extracted, and :po_files_created (files created or updated)
60
+ # @raise [Errors::CoverageError] When a locale is below `translation.min_coverage`
52
61
  def extract_site
53
- Jekyll.logger.info 'Localization', 'Extracting translatable strings...'
62
+ Jekyll.logger.info Constants::LOG_PREFIX, 'Extracting translatable strings...'
54
63
  start_time = Time.now
55
64
  stats = process_all_html_files
56
65
  @result_saver.finalize_compendia
57
66
  translate_all_compendia
58
67
  ExtractionLogger.log_summary(stats, Time.now - start_time) if show_statistics?
68
+ check_coverage
59
69
  stats
60
70
  end
61
71
 
@@ -84,20 +94,6 @@ module Jekyll
84
94
  end
85
95
  end
86
96
 
87
- def process_file(file_path)
88
- return default_stats unless @config_loader.valid_for_extraction?(file_path)
89
-
90
- config = @config_loader.load_page_config(file_path)
91
- entries = extract_strings_from_file(file_path, config)
92
- return default_stats if entries.empty?
93
-
94
- page_path = construct_page_path(file_path)
95
- @result_saver.save_results(config, entries, page_path)
96
- rescue StandardError => e
97
- ExtractionLogger.log_error(file_path, e)
98
- default_stats
99
- end
100
-
101
97
  def default_stats
102
98
  { files_processed: 0, strings_extracted: 0, po_files_created: 0 }
103
99
  end
@@ -108,32 +104,56 @@ module Jekyll
108
104
  html = FileOperations.read_utf8(file_path)
109
105
  exclude_selectors = @config_loader.extract_exclude_selectors(config)
110
106
 
111
- extractor = HtmlStringExtractor.new(config.translatable_attributes, exclude_selectors)
107
+ extractor = HtmlStringExtractor.new(config.translatable_attributes, exclude_selectors,
108
+ baseurl: @site.config['baseurl'] || '')
112
109
  extractor.extract(html, @dest, file_path)
113
110
  end
114
111
 
115
- # Machine translation uses the settings of the first localized page that
116
- # turns LibreTranslate on, including that page's `locales`.
112
+ # Machine translation is a site-wide step: it reads the `jekyll_l10n`
113
+ # settings and covers every locale the site uses.
114
+ #
115
+ # @return [PageLocalesConfig, nil] The site config, or nil when
116
+ # LibreTranslate is off
117
117
  def find_libretranslate_config
118
118
  return nil unless @site.respond_to?(:pages)
119
119
 
120
- @site.pages.each do |page|
121
- next unless page.data['with_locales'] == true
120
+ config = SiteConfigAccessor.site_config(@site, all_locales: true)
121
+ config if config.libretranslate_enabled?
122
+ end
122
123
 
123
- config = @config_loader.load_page_config(page.destination(''))
124
- return config if config.libretranslate_enabled?
125
- end
124
+ private
125
+
126
+ # per-page body called from the html_files loop in process_all_html_files
127
+ def process_file(file_path)
128
+ return default_stats unless @config_loader.valid_for_extraction?(file_path)
129
+
130
+ config = @config_loader.load_page_config(file_path)
131
+ entries = extract_strings_from_file(file_path, config)
132
+ return default_stats if entries.empty?
126
133
 
127
- nil
134
+ @extracted_msgids.merge(entries.map { |entry| entry[:msgid] })
135
+
136
+ page_path = construct_page_path(file_path)
137
+ @result_saver.save_results(config, entries, page_path)
138
+ rescue StandardError => e
139
+ ExtractionLogger.log_error(file_path, e)
140
+ default_stats
128
141
  end
129
142
 
130
- private
143
+ # Runs after machine translation, so its entries count, and outside
144
+ # ErrorHandler, so a CoverageError fails the build.
145
+ def check_coverage
146
+ config = SiteConfigAccessor.site_config(@site, all_locales: true)
147
+ report = CoverageReport.new(@site, config, @extracted_msgids)
148
+ report.warn_invalid
149
+ report.log if config.show_statistics?
150
+ report.enforce!(config.min_coverage)
151
+ end
131
152
 
132
153
  # The summary covers the whole site, so read the site-wide setting, not a
133
154
  # page's.
134
155
  def show_statistics?
135
- site_data = SiteConfigAccessor.extract_locales_data(@site)
136
- PageLocalesConfig.new('with_locales_data' => site_data).show_statistics?
156
+ SiteConfigAccessor.site_config(@site).show_statistics?
137
157
  end
138
158
 
139
159
  def construct_page_path(file_path)
@@ -1,8 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative '../constants'
3
4
  require_relative 'dom_text_extractor'
4
5
  require_relative 'dom_attribute_extractor'
5
6
  require_relative 'logger'
7
+ require_relative '../utils/baseurl_normalizer'
8
+ require_relative '../utils/html_elements'
9
+ require_relative '../utils/html_parser'
6
10
 
7
11
  module Jekyll
8
12
  module L10n
@@ -27,7 +31,7 @@ module Jekyll
27
31
  # entries = extractor.extract(html_content, '_site', 'docs/index.html')
28
32
  # # Returns array of hash entries with :msgid, :msgstr, :reference keys
29
33
  class HtmlStringExtractor
30
- attr_reader :translatable_attrs, :exclude_selectors
34
+ attr_reader :translatable_attrs, :exclude_selectors, :baseurl
31
35
 
32
36
  # Initialize a new HtmlStringExtractor.
33
37
  #
@@ -35,9 +39,12 @@ module Jekyll
35
39
  # (e.g., ['title', 'alt', 'aria-label', 'placeholder', 'aria-description'])
36
40
  # @param exclude_selectors [Array<String>] CSS selectors for elements to skip
37
41
  # during extraction (e.g., ['script', 'style', '.no-translate'])
38
- def initialize(translatable_attrs, exclude_selectors)
42
+ # @param baseurl [String] (keyword) Site baseurl, kept out of msgids
43
+ # (see {BaseurlNormalizer})
44
+ def initialize(translatable_attrs, exclude_selectors, baseurl: '')
39
45
  @translatable_attrs = translatable_attrs
40
46
  @exclude_selectors = exclude_selectors
47
+ @baseurl = baseurl
41
48
  end
42
49
 
43
50
  # Extract translatable strings from HTML.
@@ -57,7 +64,8 @@ module Jekyll
57
64
  def extract(html, dest, file_path)
58
65
  entries = []
59
66
 
60
- doc = Nokogiri::HTML(html)
67
+ doc = HtmlParser.parse_document(html)
68
+ BaseurlNormalizer.strip(doc, @baseurl)
61
69
  walk_dom(doc.root, file_path, entries, dest)
62
70
 
63
71
  entries.uniq { |e| e[:msgid] }
@@ -65,17 +73,23 @@ module Jekyll
65
73
 
66
74
  private
67
75
 
68
- def walk_dom(node, file_path, entries, dest)
76
+ # Below a content element that has a msgid, inline text is translated only
77
+ # as part of that msgid, so it gets none of its own (covered). A nested
78
+ # content element is looked up by its own msgid, so it is extracted again
79
+ # and decides coverage for its own descendants.
80
+ def walk_dom(node, file_path, entries, dest, covered: false)
69
81
  return if node.nil? || should_exclude_element?(node)
70
82
 
71
- process_text_content(node, file_path, entries, dest)
83
+ block = node.element? && HtmlElements::CONTENT_ELEMENTS.include?(node.name)
84
+ entry = process_text_content(node, file_path, entries, dest) if block || !covered
72
85
  process_attributes(node, file_path, entries, dest)
73
- process_children(node, file_path, entries, dest)
86
+ process_children(node, file_path, entries, dest, covered: block ? !entry.nil? : covered)
74
87
  end
75
88
 
76
89
  def process_text_content(node, file_path, entries, dest)
77
90
  entry = DomTextExtractor.extract(node, file_path, dest)
78
91
  entries << entry if entry
92
+ entry
79
93
  end
80
94
 
81
95
  def process_attributes(node, file_path, entries, dest)
@@ -83,8 +97,8 @@ module Jekyll
83
97
  entries.concat(attr_entries)
84
98
  end
85
99
 
86
- def process_children(node, file_path, entries, dest)
87
- node.children.each { |child| walk_dom(child, file_path, entries, dest) }
100
+ def process_children(node, file_path, entries, dest, covered: false)
101
+ node.children.each { |child| walk_dom(child, file_path, entries, dest, covered: covered) }
88
102
  end
89
103
 
90
104
  def should_exclude_element?(node)
@@ -92,10 +106,10 @@ module Jekyll
92
106
 
93
107
  @exclude_selectors.any? { |selector| node.matches?(selector) }
94
108
  rescue Nokogiri::CSS::SyntaxError => e
95
- Jekyll.logger.warn 'Localization', "CSS selector syntax error: #{e.message}"
109
+ Jekyll.logger.warn Constants::LOG_PREFIX, "CSS selector syntax error: #{e.message}"
96
110
  false
97
111
  rescue StandardError => e
98
- Jekyll.logger.warn 'Localization', "Selector matching error (continuing): #{e.message}"
112
+ Jekyll.logger.warn Constants::LOG_PREFIX, "Selector matching error (continuing): #{e.message}"
99
113
  false
100
114
  end
101
115
  end
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative '../constants'
3
4
  require_relative '../utils/error_handler'
4
5
 
5
6
  module Jekyll
@@ -39,11 +40,11 @@ module Jekyll
39
40
  # @param elapsed [Float] Time elapsed in seconds
40
41
  # @return [void]
41
42
  def self.log_summary(stats, elapsed)
42
- Jekyll.logger.info 'Localization', 'Extraction complete:'
43
- Jekyll.logger.info 'Localization', " Files processed: #{stats[:files_processed]}"
44
- Jekyll.logger.info 'Localization', " Strings extracted: #{stats[:strings_extracted]}"
45
- Jekyll.logger.info 'Localization', " PO files created/updated: #{stats[:po_files_created]}"
46
- Jekyll.logger.info 'Localization', " Time: #{elapsed.round(2)}s"
43
+ Jekyll.logger.info Constants::LOG_PREFIX, 'Extraction complete:'
44
+ Jekyll.logger.info Constants::LOG_PREFIX, " Files processed: #{stats[:files_processed]}"
45
+ Jekyll.logger.info Constants::LOG_PREFIX, " Strings extracted: #{stats[:strings_extracted]}"
46
+ Jekyll.logger.info Constants::LOG_PREFIX, " PO files created/updated: #{stats[:po_files_created]}"
47
+ Jekyll.logger.info Constants::LOG_PREFIX, " Time: #{elapsed.round(2)}s"
47
48
  end
48
49
  end
49
50
  end
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative '../configuration'
3
4
  require_relative '../po_file/manager'
4
5
  require_relative '../utils/page_locales_config'
5
6
  require_relative '../utils/site_config_accessor'
@@ -33,8 +34,7 @@ module Jekyll
33
34
  # @param site [Jekyll::Site] Jekyll site object
34
35
  def initialize(site)
35
36
  @site = site
36
- with_locales_data = SiteConfigAccessor.extract_locales_data(@site)
37
- @site_config = PageLocalesConfig.new({ 'with_locales_data' => with_locales_data })
37
+ @site_config = SiteConfigAccessor.site_config(@site)
38
38
  end
39
39
 
40
40
  # Save extraction results to PO files.
@@ -105,17 +105,12 @@ module Jekyll
105
105
  po_files_created
106
106
  end
107
107
 
108
- # Build a PageLocalesConfig whose locale list is the union of site-wide
109
- # defaults and every page-level front-matter locale override.
110
- #
111
- # All other site-wide settings (locales_dir, extraction, translation,
112
- # logging, etc.) are preserved unchanged; only the locale list is widened.
108
+ # The site config with every locale any page uses, not only the
109
+ # site-wide list (see SiteConfigAccessor.all_configured_locales).
113
110
  #
114
111
  # @return [PageLocalesConfig]
115
112
  def merge_config
116
- all_locales = SiteConfigAccessor.all_configured_locales(@site)
117
- merged_data = (@site_config.data['with_locales_data'] || {}).merge('locales' => all_locales)
118
- PageLocalesConfig.new({ 'with_locales_data' => merged_data })
113
+ SiteConfigAccessor.site_config(@site, all_locales: true)
119
114
  end
120
115
 
121
116
  private :save_po_files