jekyll-l10n 1.7.1 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +44 -0
  3. data/lib/jekyll-l10n/configuration/schema.rb +308 -0
  4. data/lib/jekyll-l10n/configuration.rb +165 -0
  5. data/lib/jekyll-l10n/constants.rb +45 -10
  6. data/lib/jekyll-l10n/errors.rb +7 -0
  7. data/lib/jekyll-l10n/extraction/compendium_merger.rb +9 -11
  8. data/lib/jekyll-l10n/extraction/compendium_translator.rb +27 -11
  9. data/lib/jekyll-l10n/extraction/config_loader.rb +28 -58
  10. data/lib/jekyll-l10n/extraction/coverage_report.rb +129 -0
  11. data/lib/jekyll-l10n/extraction/dom_text_extractor.rb +2 -31
  12. data/lib/jekyll-l10n/extraction/extractor.rb +72 -65
  13. data/lib/jekyll-l10n/extraction/html_string_extractor.rb +24 -10
  14. data/lib/jekyll-l10n/extraction/logger.rb +6 -5
  15. data/lib/jekyll-l10n/extraction/result_saver.rb +5 -10
  16. data/lib/jekyll-l10n/instrumentation.rb +172 -139
  17. data/lib/jekyll-l10n/jekyll/generator.rb +47 -61
  18. data/lib/jekyll-l10n/jekyll/localized_page.rb +27 -49
  19. data/lib/jekyll-l10n/jekyll/post_write_html_reprocessor.rb +53 -48
  20. data/lib/jekyll-l10n/jekyll/post_write_processor.rb +2 -1
  21. data/lib/jekyll-l10n/jekyll/regeneration_checker.rb +70 -50
  22. data/lib/jekyll-l10n/jekyll/url_filter.rb +13 -13
  23. data/lib/jekyll-l10n/po_file/entry_parser.rb +155 -0
  24. data/lib/jekyll-l10n/po_file/fuzzy_matcher.rb +67 -53
  25. data/lib/jekyll-l10n/po_file/loader.rb +20 -11
  26. data/lib/jekyll-l10n/po_file/manager.rb +32 -17
  27. data/lib/jekyll-l10n/po_file/merger.rb +2 -1
  28. data/lib/jekyll-l10n/po_file/path_builder.rb +11 -1
  29. data/lib/jekyll-l10n/po_file/reader.rb +40 -508
  30. data/lib/jekyll-l10n/po_file/writer.rb +31 -16
  31. data/lib/jekyll-l10n/translation/block_text_extractor.rb +3 -7
  32. data/lib/jekyll-l10n/translation/html_translator.rb +136 -157
  33. data/lib/jekyll-l10n/translation/libre_translate_languages.rb +73 -0
  34. data/lib/jekyll-l10n/translation/libre_translator.rb +134 -78
  35. data/lib/jekyll-l10n/translation/page_translation_loader.rb +13 -29
  36. data/lib/jekyll-l10n/translation/request_slots.rb +69 -0
  37. data/lib/jekyll-l10n/translation/translation_progress.rb +77 -0
  38. data/lib/jekyll-l10n/translation/translator.rb +49 -78
  39. data/lib/jekyll-l10n/utils/baseurl_normalizer.rb +108 -0
  40. data/lib/jekyll-l10n/utils/debug_logger.rb +4 -51
  41. data/lib/jekyll-l10n/utils/error_handler.rb +9 -24
  42. data/lib/jekyll-l10n/utils/external_link_icon_preserver.rb +5 -62
  43. data/lib/jekyll-l10n/utils/html_parser.rb +24 -31
  44. data/lib/jekyll-l10n/utils/html_text_utils.rb +146 -91
  45. data/lib/jekyll-l10n/utils/logger_formatter.rb +6 -4
  46. data/lib/jekyll-l10n/utils/msgstr_validator.rb +74 -0
  47. data/lib/jekyll-l10n/utils/page_locales_config.rb +165 -223
  48. data/lib/jekyll-l10n/utils/po_entry_converter.rb +0 -43
  49. data/lib/jekyll-l10n/utils/site_config_accessor.rb +82 -42
  50. data/lib/jekyll-l10n/utils/translation_resolver.rb +56 -6
  51. data/lib/jekyll-l10n/utils/url_path_builder.rb +5 -27
  52. data/lib/jekyll-l10n/utils/url_transformer.rb +5 -11
  53. data/lib/jekyll-l10n/version.rb +1 -1
  54. data/lib/jekyll-l10n.rb +67 -176
  55. metadata +12 -3
@@ -26,92 +26,106 @@ module Jekyll
26
26
  #
27
27
  # Skips new_msgid values longer than MAX_FUZZY_MSGID_LENGTH (long strings are
28
28
  # unique HTML fragments with no useful near-duplicate and Levenshtein is O(n²)).
29
- # Pre-filters candidates to the length range where similarity ≥ threshold is
30
- # mathematically possible before invoking Levenshtein.
29
+ # Only candidates whose length allows a similarity ≥ threshold are scored, and
30
+ # each one stops being scored as soon as its edit distance is too large.
31
31
  #
32
32
  # @param new_msgid [String] the new source string to match
33
33
  # @param candidates [Hash] { old_msgid => entry } where entry is either a
34
34
  # String msgstr or a Hash with :msgstr key
35
35
  # @param threshold [Float] minimum similarity score to accept (0.0–1.0)
36
+ # @param scores [Hash, nil] cache of scores by [new_msgid, old_msgid], shared
37
+ # between calls that use the same threshold, such as the locales of one
38
+ # compendium merge. Without it, every call scores its candidates again.
36
39
  # @return [Hash, nil] { msgid: String, msgstr: String } or nil if no match
37
- def self.find_match(new_msgid, candidates, threshold: THRESHOLD)
40
+ def self.find_match(new_msgid, candidates, threshold: THRESHOLD, scores: nil)
38
41
  return nil if candidates.empty?
39
42
  return nil if new_msgid.nil? || new_msgid.length > Constants::MAX_FUZZY_MSGID_LENGTH
40
43
 
41
- len = new_msgid.length
42
- min_feas = (len * threshold).ceil
43
- max_feas = threshold.positive? ? (len / threshold).floor : Float::INFINITY
44
-
45
- best = best_candidate(new_msgid, candidates, min_feas, max_feas, threshold)
46
- return nil unless best
47
-
48
- { msgid: best[:msgid], msgstr: msgstr_from_entry(best[:entry]) }
44
+ best = best_candidate(new_msgid, candidates, threshold, scores)
45
+ best && { msgid: best[:msgid], msgstr: msgstr_from_entry(best[:entry]) }
49
46
  end
50
47
 
51
- def self.best_candidate(new_msgid, candidates, min_feas, max_feas, threshold)
52
- best_msgid = best_entry = nil
48
+ def self.best_candidate(new_msgid, candidates, threshold, scores)
49
+ lengths = feasible_lengths(new_msgid.length, threshold)
50
+ best = nil
53
51
  best_score = 0.0
54
52
 
55
53
  candidates.each do |old_msgid, entry|
56
- ol = old_msgid.length
57
- next if ol < min_feas || ol > max_feas
54
+ next unless lengths.cover?(old_msgid.length)
58
55
 
59
- score = similarity(new_msgid, old_msgid)
56
+ score = cached_score(new_msgid, old_msgid, threshold, scores)
60
57
  next unless score > best_score && score >= threshold
61
58
 
62
59
  best_score = score
63
- best_msgid = old_msgid
64
- best_entry = entry
60
+ best = { msgid: old_msgid, entry: entry }
65
61
  break if best_score >= 1.0
66
62
  end
63
+ best
64
+ end
67
65
 
68
- best_msgid ? { msgid: best_msgid, entry: best_entry } : nil
66
+ # Candidate lengths for which a similarity ≥ threshold is possible.
67
+ def self.feasible_lengths(length, threshold)
68
+ max = threshold.positive? ? (length / threshold).floor : Float::INFINITY
69
+ (length * threshold).ceil..max
69
70
  end
70
71
 
71
- # Normalized Levenshtein similarity between two strings.
72
- #
73
- # Returns 0.0 immediately when the length ratio falls below the threshold —
74
- # the maximum achievable similarity is min_len/max_len, so Levenshtein cannot
75
- # produce a useful result and the O(n²) computation is skipped.
76
- #
77
- # @param str_a [String]
78
- # @param str_b [String]
79
- # @return [Float] 0.0 (completely different) to 1.0 (identical)
80
- def self.similarity(str_a, str_b)
72
+ def self.cached_score(new_msgid, old_msgid, threshold, scores)
73
+ return bounded_similarity(new_msgid, old_msgid, threshold) unless scores
74
+
75
+ scores.fetch([new_msgid, old_msgid]) do |key|
76
+ scores[key] = bounded_similarity(new_msgid, old_msgid, threshold)
77
+ end
78
+ end
79
+
80
+ # Normalized Levenshtein similarity (0.0 to 1.0), or 0.0 as soon as the
81
+ # score is certain to be below threshold, without finishing the edit
82
+ # distance.
83
+ def self.bounded_similarity(str_a, str_b, threshold)
81
84
  return 1.0 if str_a == str_b
82
- return 0.0 if str_a.empty? || str_b.empty?
83
85
 
84
86
  max_len = [str_a.length, str_b.length].max
85
- min_len = [str_a.length, str_b.length].min
86
- return 0.0 if min_len.to_f / max_len < THRESHOLD
87
+ # Rounded up so a pair exactly at the threshold is never cut off.
88
+ limit = (max_len * (1 - threshold)).ceil
89
+ return 0.0 if char_count_distance(str_a, str_b) > limit
87
90
 
88
- dist = levenshtein(str_a, str_b)
89
- 1.0 - (dist.to_f / max_len)
91
+ distance = bounded_distance(str_a, str_b, limit)
92
+ distance > limit ? 0.0 : 1.0 - (distance.to_f / max_len)
90
93
  end
91
94
 
92
- def self.levenshtein(str_a, str_b)
93
- # Keep the shorter string as the row to minimise the dp array size.
94
- str_a, str_b = str_b, str_a if str_a.length > str_b.length
95
+ # A lower bound on the Levenshtein distance, from character counts alone:
96
+ # each edit changes the counts by at most one on each side.
97
+ def self.char_count_distance(str_a, str_b)
98
+ diff = str_a.each_char.tally
99
+ str_b.each_char { |char| diff[char] = diff.fetch(char, 0) - 1 }
100
+ surplus, deficit = diff.values.partition(&:positive?)
101
+ [surplus.sum, -deficit.sum].max
102
+ end
95
103
 
104
+ # Levenshtein distance, or any value above limit once every way to finish
105
+ # the comparison already costs more than limit.
106
+ def self.bounded_distance(str_a, str_b, limit)
107
+ # Keep the shorter string as the row to minimise the row size.
108
+ str_a, str_b = str_b, str_a if str_a.length > str_b.length
96
109
  a_chars = str_a.chars
97
- b_chars = str_b.chars
98
- m = a_chars.length
99
- n = b_chars.length
100
-
101
- curr = Array.new(m + 1) { |i| i }
102
- prev = Array.new(m + 1, 0)
103
-
104
- n.times do |j|
105
- curr, prev = prev, curr # swap in place — no allocation
106
- curr[0] = j + 1
107
- m.times do |i|
108
- cost = a_chars[i] == b_chars[j] ? 0 : 1
109
- curr[i + 1] = [curr[i] + 1, prev[i + 1] + 1, prev[i] + cost].min
110
- end
110
+ row = (0..a_chars.length).to_a
111
+
112
+ str_b.each_char.with_index(1) do |b_char, row_number|
113
+ row = next_row(row, a_chars, b_char, row_number)
114
+ return limit + 1 if row.min > limit
115
+ end
116
+ row.last
117
+ end
118
+
119
+ def self.next_row(prev, a_chars, b_char, row_number)
120
+ curr = [row_number]
121
+ a_chars.each_with_index do |a_char, i|
122
+ cost = a_char == b_char ? 0 : 1
123
+ curr << [curr[i] + 1, prev[i + 1] + 1, prev[i] + cost].min
111
124
  end
112
- curr[m]
125
+ curr
113
126
  end
114
- private_class_method :levenshtein, :best_candidate
127
+ private_class_method :best_candidate, :feasible_lengths, :cached_score,
128
+ :bounded_similarity, :char_count_distance, :bounded_distance, :next_row
115
129
  end
116
130
  end
117
131
  end
@@ -1,9 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative 'reader'
4
- require_relative 'manager'
5
4
  require_relative 'path_builder'
6
5
  require_relative '../utils/error_handler'
6
+ require_relative '../utils/msgstr_validator'
7
7
 
8
8
  module Jekyll
9
9
  module L10n
@@ -18,6 +18,7 @@ module Jekyll
18
18
  # * Construct file paths for locale/page combinations
19
19
  # * Load PO files from disk when not cached
20
20
  # * Parse PO file content into translation hashes
21
+ # * Drop entries whose msgstr markup does not match the msgid (MsgstrValidator)
21
22
  # * Cache parsed translations for reuse
22
23
  # * Handle file-not-found gracefully (return empty hash)
23
24
  # * Log errors during loading
@@ -26,11 +27,19 @@ module Jekyll
26
27
  # translations = PoFileLoader.load('_site', '_locales', 'es', 'docs/index.html')
27
28
  # # Returns cached or newly-loaded translations for that page/locale
28
29
  #
29
- # @see Jekyll::L10n::PoFileManager for caching implementation details
30
+ # @see Jekyll::L10n::PoFileManager for the public cache API
30
31
  class PoFileLoader
32
+ # The global PO file cache, mapping PO file paths to translation
33
+ # hashes. Read and clear it through PoFileManager.
34
+ #
35
+ # @return [Hash]
36
+ def self.cache
37
+ @cache ||= {}
38
+ end
39
+
31
40
  # Load a PO file with caching.
32
41
  #
33
- # First checks the global PoFileManager cache for already-loaded translations.
42
+ # First checks the global cache for already-loaded translations.
34
43
  # If not found, constructs the file path, loads from disk if file exists,
35
44
  # parses the PO file, caches the result, and returns. Returns empty hash
36
45
  # if file doesn't exist or if an error occurs during loading.
@@ -42,24 +51,24 @@ module Jekyll
42
51
  # If nil, loads compendium.
43
52
  # @return [Hash] Translation hash { msgid => msgstr } or empty hash if not found
44
53
  def self.load(source, locales_dir, locale, page_path)
45
- cache_key = "#{locale}:#{page_path}"
46
- cached = PoFileManager.cache[cache_key]
47
- return cached if cached
48
-
49
54
  po_path = PoPathBuilder.build(source, locales_dir, locale, page_path)
55
+ cached = cache[po_path]
56
+ return cached if cached
50
57
  return {} unless File.exist?(po_path)
51
58
 
52
- load_and_cache(cache_key, po_path)
59
+ load_and_cache(po_path)
53
60
  end
54
61
 
55
- def self.load_and_cache(cache_key, po_path)
56
- translations = PoFileReader.parse_for_translation(po_path)
57
- PoFileManager.cache[cache_key] = translations
62
+ def self.load_and_cache(po_path)
63
+ translations = MsgstrValidator.reject_invalid(PoFileReader.parse_for_translation(po_path), po_path)
64
+ cache[po_path] = translations
58
65
  translations
59
66
  rescue StandardError => e
60
67
  ErrorHandler.log_warning("loading PO file #{po_path}", e)
61
68
  {}
62
69
  end
70
+
71
+ private_class_method :load_and_cache
63
72
  end
64
73
  end
65
74
  end
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative '../constants'
3
4
  require_relative 'loader'
4
5
  require_relative 'writer'
5
6
  require_relative 'merger'
@@ -22,7 +23,7 @@ module Jekyll
22
23
  # * Load and cache PO files (page-specific and compendium)
23
24
  # * Save new or updated PO files with optional merging of existing translations
24
25
  # * Merge PO files for locales (combining page-specific with compendium)
25
- # * Manage global in-memory cache keyed by locale and page path
26
+ # * Manage global in-memory cache keyed by PO file path
26
27
  # * Clear cache when files are written or rebuilt
27
28
  # * Construct proper file paths for locale/page combinations
28
29
  #
@@ -31,25 +32,26 @@ module Jekyll
31
32
  # translations = manager.load_po_file('es', 'docs/index.html')
32
33
  # manager.save_po_file('es', entries, page_path: 'docs/index.html')
33
34
  class PoFileManager
34
- DEFAULT_LOCALES_DIR = '_locales'
35
+ DEFAULT_LOCALES_DIR = Constants::DEFAULT_LOCALES_DIR
35
36
  attr_reader :site, :locales_dir
36
37
 
37
38
  class << self
38
39
  # Access the global PO file cache.
39
40
  #
40
- # @return [Hash] Cache mapping "locale:page_path" keys to translation hashes
41
+ # @return [Hash] Cache mapping PO file paths to translation hashes
41
42
  def cache
42
- @cache ||= {}
43
+ PoFileLoader.cache
43
44
  end
44
45
 
45
46
  # Clear the global PO file cache.
46
47
  #
47
- # Clears all cached PO file translations. Should be called when the site
48
- # is rebuilt or PO files are modified to ensure fresh data is loaded.
48
+ # Clears all cached PO file translations. Runs on every site reset (see
49
+ # the hook at the end of this file), so each rebuild reloads PO files
50
+ # edited since the previous one.
49
51
  #
50
52
  # @return [void]
51
53
  def clear_cache
52
- @cache = {}
54
+ PoFileLoader.cache.clear
53
55
  end
54
56
 
55
57
  # Get the current size of the global cache.
@@ -64,7 +66,7 @@ module Jekyll
64
66
  #
65
67
  # @param site [Jekyll::Site] Jekyll site object
66
68
  # @param locales_dir [String] Directory containing PO files (defaults to
67
- # DEFAULT_LOCALES_DIR = "_locales")
69
+ # Constants::DEFAULT_LOCALES_DIR)
68
70
  def initialize(site, locales_dir = DEFAULT_LOCALES_DIR)
69
71
  @site = site
70
72
  @source = SiteConfigAccessor.source(@site)
@@ -98,6 +100,23 @@ module Jekyll
98
100
  PoFileLoader.load(@source, @locales_dir, locale, page_path)
99
101
  end
100
102
 
103
+ # Path of a locale's compendium in this manager's locales directory.
104
+ #
105
+ # @param locale [String] Target locale code (e.g., 'es', 'fr')
106
+ # @return [String] Full path to the compendium PO file
107
+ def compendium_path(locale)
108
+ PoPathBuilder.build(@source, @locales_dir, locale, nil)
109
+ end
110
+
111
+ # Directory holding a locale's page-specific PO files in this manager's
112
+ # locales directory.
113
+ #
114
+ # @param locale [String] Target locale code (e.g., 'es', 'fr')
115
+ # @return [String] Full path to the locale directory
116
+ def locale_dir(locale)
117
+ PoPathBuilder.locale_dir(@source, @locales_dir, locale)
118
+ end
119
+
101
120
  # Save a PO file for a locale and optional page.
102
121
  #
103
122
  # Writes entries to PO file, optionally merging with existing translations to
@@ -111,8 +130,7 @@ module Jekyll
111
130
  # @return [Boolean] True if write successful, false on error
112
131
  def save_po_file(locale, entries, page_path: nil, skip_merge: false)
113
132
  po_path = PoPathBuilder.build(@source, @locales_dir, locale, page_path)
114
- prepare_and_write_po_file(po_path, entries, locale, page_path: page_path,
115
- skip_merge: skip_merge)
133
+ prepare_and_write_po_file(po_path, entries, locale, skip_merge: skip_merge)
116
134
  rescue StandardError => e
117
135
  ErrorHandler.log_error("saving PO file #{po_path}", e)
118
136
  false
@@ -143,19 +161,16 @@ module Jekyll
143
161
 
144
162
  private
145
163
 
146
- def prepare_and_write_po_file(po_path, entries, locale, page_path: nil, skip_merge: false)
164
+ def prepare_and_write_po_file(po_path, entries, locale, skip_merge: false)
147
165
  LoggerFormatter.debug_if_enabled('PoFileManager',
148
166
  "Writing PO file: #{po_path} (#{entries.length} entries)")
149
167
  FileOperations.ensure_directory(po_path)
150
168
  PoFileWriter.write(po_path, entries, locale, skip_merge: skip_merge)
151
- invalidate_po_cache(locale, page_path)
169
+ self.class.cache.delete(po_path)
152
170
  true
153
171
  end
154
-
155
- def invalidate_po_cache(locale, page_path)
156
- cache_key = "#{locale}:#{page_path}"
157
- self.class.cache.delete(cache_key)
158
- end
159
172
  end
160
173
  end
161
174
  end
175
+
176
+ Jekyll::Hooks.register(:site, :after_reset) { Jekyll::L10n::PoFileManager.clear_cache }
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative 'path_builder'
3
4
  require_relative 'reader'
4
5
  require_relative '../utils/error_handler'
5
6
 
@@ -37,7 +38,7 @@ module Jekyll
37
38
  # @return [Hash<String, Hash>] Merged translation hash where keys are msgid strings
38
39
  # and values are hashes with :msgstr and :reference keys
39
40
  def self.merge_for_locale(source, locales_dir, locale)
40
- locale_dir = File.join(source, locales_dir, locale)
41
+ locale_dir = PoPathBuilder.locale_dir(source, locales_dir, locale)
41
42
  return {} unless File.directory?(locale_dir)
42
43
 
43
44
  po_files = Dir.glob(File.join(locale_dir, '**', '*.po')).sort
@@ -40,7 +40,7 @@ module Jekyll
40
40
  File.join(source, locales_dir, "#{locale}.po")
41
41
  else
42
42
  # Build the normal path
43
- normal_path = File.join(source, locales_dir, locale, "#{page_path}.po")
43
+ normal_path = File.join(locale_dir(source, locales_dir, locale), "#{page_path}.po")
44
44
 
45
45
  # Validate against path traversal using expanded paths
46
46
  base_path = File.expand_path(File.join(source, locales_dir))
@@ -57,6 +57,16 @@ module Jekyll
57
57
  normal_path
58
58
  end
59
59
  end
60
+
61
+ # Build the directory that holds a locale's page-specific PO files.
62
+ #
63
+ # @param source [String] Site source directory
64
+ # @param locales_dir [String] Locales directory name (e.g., '_locales')
65
+ # @param locale [String] Locale code (e.g., 'es', 'fr')
66
+ # @return [String] {source}/{locales_dir}/{locale}
67
+ def self.locale_dir(source, locales_dir, locale)
68
+ File.join(source, locales_dir, locale)
69
+ end
60
70
  end
61
71
  end
62
72
  end