jekyll-l10n 2.0.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +44 -0
  3. data/lib/jekyll-l10n/configuration/schema.rb +308 -0
  4. data/lib/jekyll-l10n/configuration.rb +165 -0
  5. data/lib/jekyll-l10n/constants.rb +45 -10
  6. data/lib/jekyll-l10n/errors.rb +7 -0
  7. data/lib/jekyll-l10n/extraction/compendium_merger.rb +9 -11
  8. data/lib/jekyll-l10n/extraction/compendium_translator.rb +27 -11
  9. data/lib/jekyll-l10n/extraction/config_loader.rb +4 -4
  10. data/lib/jekyll-l10n/extraction/coverage_report.rb +129 -0
  11. data/lib/jekyll-l10n/extraction/dom_text_extractor.rb +2 -31
  12. data/lib/jekyll-l10n/extraction/extractor.rb +48 -28
  13. data/lib/jekyll-l10n/extraction/html_string_extractor.rb +24 -10
  14. data/lib/jekyll-l10n/extraction/logger.rb +6 -5
  15. data/lib/jekyll-l10n/extraction/result_saver.rb +5 -10
  16. data/lib/jekyll-l10n/instrumentation.rb +172 -139
  17. data/lib/jekyll-l10n/jekyll/generator.rb +10 -22
  18. data/lib/jekyll-l10n/jekyll/localized_page.rb +27 -49
  19. data/lib/jekyll-l10n/jekyll/post_write_html_reprocessor.rb +42 -25
  20. data/lib/jekyll-l10n/jekyll/post_write_processor.rb +2 -1
  21. data/lib/jekyll-l10n/jekyll/regeneration_checker.rb +16 -10
  22. data/lib/jekyll-l10n/jekyll/url_filter.rb +8 -7
  23. data/lib/jekyll-l10n/po_file/entry_parser.rb +155 -0
  24. data/lib/jekyll-l10n/po_file/fuzzy_matcher.rb +67 -53
  25. data/lib/jekyll-l10n/po_file/loader.rb +20 -11
  26. data/lib/jekyll-l10n/po_file/manager.rb +32 -17
  27. data/lib/jekyll-l10n/po_file/merger.rb +2 -1
  28. data/lib/jekyll-l10n/po_file/path_builder.rb +11 -1
  29. data/lib/jekyll-l10n/po_file/reader.rb +40 -508
  30. data/lib/jekyll-l10n/po_file/writer.rb +31 -16
  31. data/lib/jekyll-l10n/translation/block_text_extractor.rb +3 -7
  32. data/lib/jekyll-l10n/translation/html_translator.rb +136 -157
  33. data/lib/jekyll-l10n/translation/libre_translate_languages.rb +73 -0
  34. data/lib/jekyll-l10n/translation/libre_translator.rb +134 -78
  35. data/lib/jekyll-l10n/translation/request_slots.rb +69 -0
  36. data/lib/jekyll-l10n/translation/translation_progress.rb +77 -0
  37. data/lib/jekyll-l10n/translation/translator.rb +20 -17
  38. data/lib/jekyll-l10n/utils/baseurl_normalizer.rb +108 -0
  39. data/lib/jekyll-l10n/utils/debug_logger.rb +4 -51
  40. data/lib/jekyll-l10n/utils/error_handler.rb +9 -24
  41. data/lib/jekyll-l10n/utils/external_link_icon_preserver.rb +5 -62
  42. data/lib/jekyll-l10n/utils/html_parser.rb +24 -31
  43. data/lib/jekyll-l10n/utils/html_text_utils.rb +146 -91
  44. data/lib/jekyll-l10n/utils/logger_formatter.rb +6 -4
  45. data/lib/jekyll-l10n/utils/msgstr_validator.rb +74 -0
  46. data/lib/jekyll-l10n/utils/page_locales_config.rb +126 -107
  47. data/lib/jekyll-l10n/utils/po_entry_converter.rb +0 -43
  48. data/lib/jekyll-l10n/utils/site_config_accessor.rb +56 -54
  49. data/lib/jekyll-l10n/utils/translation_resolver.rb +56 -6
  50. data/lib/jekyll-l10n/utils/url_path_builder.rb +5 -27
  51. data/lib/jekyll-l10n/utils/url_transformer.rb +5 -11
  52. data/lib/jekyll-l10n/version.rb +1 -1
  53. data/lib/jekyll-l10n.rb +9 -6
  54. metadata +12 -3
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative '../configuration'
4
+ require_relative '../constants'
3
5
  require_relative '../utils/site_config_accessor'
4
6
  require_relative '../po_file/path_builder'
5
7
  require_relative '../utils/locale_url_formatter'
@@ -30,13 +32,10 @@ module Jekyll
30
32
  # end
31
33
  #
32
34
  class RegenerationChecker
33
- # @!visibility private
34
- DEFAULT_LOCALES_DIR = '_locales'
35
-
36
35
  # @param site [Jekyll::Site] The Jekyll site
37
36
  # @param locales_dir [String] Directory, relative to the site source, that
38
- # holds the PO files. Defaults to DEFAULT_LOCALES_DIR.
39
- def initialize(site, locales_dir = DEFAULT_LOCALES_DIR)
37
+ # holds the PO files. Defaults to Constants::DEFAULT_LOCALES_DIR.
38
+ def initialize(site, locales_dir = Constants::DEFAULT_LOCALES_DIR)
40
39
  @site = site
41
40
  @locales_dir = locales_dir
42
41
  @source = SiteConfigAccessor.source(@site)
@@ -79,12 +78,17 @@ module Jekyll
79
78
  #
80
79
  # This must produce the same path as LocalizedPage#destination.
81
80
  #
81
+ # It is the English page's own output path, as Jekyll computes it, under
82
+ # the locale segment, since the localized page's URL is the English URL
83
+ # with that segment in front (LocalizedPage#url).
84
+ #
82
85
  # @param page [Jekyll::Page] The source (English) page
83
86
  # @param locale [String] Locale code, e.g. 'es' or 'pt_BR'
84
- # @return [String] Path such as "es/about/index.html" (no leading slash)
87
+ # @return [String] Path such as "es/about/index.html" or "es/about.html"
88
+ # (no leading slash)
85
89
  def relative_dest_path(page, locale)
86
- url_locale = LocaleUrlFormatter.to_url_segment(locale)
87
- "#{url_locale}#{page.url.chomp('/')}/index.html"
90
+ output_path = page.destination(@destination).delete_prefix(@destination)
91
+ "#{LocaleUrlFormatter.to_url_segment(locale)}#{output_path}"
88
92
  end
89
93
 
90
94
  private
@@ -92,11 +96,13 @@ module Jekyll
92
96
  def incremental_enabled?
93
97
  return false if @site&.config.nil?
94
98
 
95
- @site.config.dig('jekyll_l10n', 'incremental') == true
99
+ SiteConfigAccessor.site_settings(@site)['incremental'] == true
96
100
  end
97
101
 
102
+ # A directory at the path is the layout of an earlier release
103
+ # (es/nav.html/index.html), so the page is written again.
98
104
  def dest_path_exists?(page, locale)
99
- File.exist?(dest_path(page, locale))
105
+ File.file?(dest_path(page, locale))
100
106
  end
101
107
 
102
108
  def dest_path(page, locale)
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'liquid'
4
+ require_relative '../configuration'
4
5
  require_relative '../constants'
5
6
  require_relative '../utils/error_handler'
6
7
  require_relative '../utils/locale_name_resolver'
@@ -27,7 +28,7 @@ module Jekyll
27
28
  # @example Usage in Jekyll Liquid templates
28
29
  # <!-- Add locale prefix to URL -->
29
30
  # <a href="{{ '/about/' | locale_url: 'es_ES' }}">Acerca de</a>
30
- # <!-- Produces: /es-ES/about/ -->
31
+ # <!-- Produces: /es-es/about/ -->
31
32
  #
32
33
  # <!-- Switch to different locale variant -->
33
34
  # <a href="{{ page.url | switch_locale_url: 'fr' }}">Français</a>
@@ -41,7 +42,7 @@ module Jekyll
41
42
  # Generate a locale-prefixed URL for the given URL and locale
42
43
  #
43
44
  # Adds a locale prefix in BCP 47 hyphenated form to the given URL. For example,
44
- # '/about/' becomes '/es-ES/about/' for locale 'es_ES'. The filter handles
45
+ # '/about/' becomes '/es-es/about/' for locale 'es_ES'. The filter handles
45
46
  # various edge cases and only applies the prefix when appropriate.
46
47
  #
47
48
  # The method performs several checks to determine if prefixing is appropriate:
@@ -56,7 +57,7 @@ module Jekyll
56
57
  # prefixing is not appropriate
57
58
  # @example
58
59
  # <!-- Explicit underscore locale converted to hyphen in URL -->
59
- # {{ '/about/' | locale_url: 'es_ES' }} # => '/es-ES/about/'
60
+ # {{ '/about/' | locale_url: 'es_ES' }} # => '/es-es/about/'
60
61
  #
61
62
  # <!-- Current page's locale -->
62
63
  # {{ '/about/' | locale_url }} # => '/es/about/' (if page locale is 'es')
@@ -71,7 +72,7 @@ module Jekyll
71
72
  # Generate a URL for the given locale variant of the current page
72
73
  #
73
74
  # Switches the current URL to a different locale variant. For example, if the current
74
- # page is '/es-ES/about/', calling this with 'fr' returns '/fr/about/'. This is useful
75
+ # page is '/es-es/about/', calling this with 'fr' returns '/fr/about/'. This is useful
75
76
  # for building language switcher menus.
76
77
  #
77
78
  # This filter first extracts the base URL (without locale prefix) from the given URL,
@@ -81,7 +82,7 @@ module Jekyll
81
82
  # @param target_locale [String] The target locale code (e.g., 'es_ES', 'fr', 'pt_BR')
82
83
  # @return [String] The URL for the target locale, or original URL if locale is invalid
83
84
  # @example
84
- # <!-- Current page is /es-ES/about/, switch to French -->
85
+ # <!-- Current page is /es-es/about/, switch to French -->
85
86
  # {{ page.url | switch_locale_url: 'fr' }} # => '/fr/about/'
86
87
  #
87
88
  # <!-- Build a language switcher -->
@@ -117,7 +118,7 @@ module Jekyll
117
118
  # @example
118
119
  # {{ page.url | locale_menu }}
119
120
  # # => [{"code"=>"en","name"=>"English","url"=>"/about/","current"=>false},
120
- # # {"code"=>"es_ES","name"=>"Spanish (Spain)","url"=>"/es-ES/about/","current"=>true}]
121
+ # # {"code"=>"es_ES","name"=>"Spanish (Spain)","url"=>"/es-es/about/","current"=>true}]
121
122
  def locale_menu(url)
122
123
  menu_locales.map do |locale|
123
124
  {
@@ -207,7 +208,7 @@ module Jekyll
207
208
 
208
209
  if page
209
210
  locales = if page.is_a?(Hash)
210
- page.dig('with_locales_data', 'locales')
211
+ Configuration::Schema.normalize(page[Configuration::PAGE_DATA_KEY], source: nil)['locales']
211
212
  else
212
213
  SiteConfigAccessor.page_locales_data(page)['locales']
213
214
  end
@@ -0,0 +1,155 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Jekyll
4
+ module L10n
5
+ # Parses the text of a GNU Gettext PO file into a translation hash.
6
+ #
7
+ # Each entry is read in three steps, in this order:
8
+ # 1. The comment lines above `msgid`: reference (`#:`), fuzzy flag (`#,`)
9
+ # and previous msgid (`#|`).
10
+ # 2. The `msgid` and `msgstr` fields, each with its continuation lines.
11
+ # 3. The stored value, whose shape depends on the mode.
12
+ #
13
+ # Modes and the value stored for each msgid:
14
+ # * :simple — the msgstr String
15
+ # * :references — { msgstr:, reference: } (reference only when present)
16
+ # * :merge — { msgstr:, reference:, fuzzy:, previous_msgid: }
17
+ #
18
+ # Entries with an empty msgid (the header) are skipped.
19
+ #
20
+ # @example
21
+ # PoEntryParser.parse(%(msgid "Hello"\nmsgstr "Hola"\n), :simple)
22
+ # # => { "Hello" => "Hola" }
23
+ class PoEntryParser
24
+ MODES = %i[simple references merge].freeze
25
+ MSGID_PATTERN = /^msgid ['"](.*)['"] *$/.freeze
26
+ MSGSTR_PATTERN = /^msgstr ['"](.*)['"] *$/.freeze
27
+
28
+ # @param content [String] PO file text
29
+ # @param mode [Symbol] One of MODES
30
+ # @return [Hash] Translations keyed by msgid
31
+ def self.parse(content, mode)
32
+ raise ArgumentError, "unknown PO parse mode: #{mode.inspect}" unless MODES.include?(mode)
33
+
34
+ new(content.to_s.split("\n"), mode).parse
35
+ end
36
+
37
+ def initialize(lines, mode)
38
+ @lines = lines
39
+ @mode = mode
40
+ end
41
+
42
+ # @return [Hash] Translations keyed by msgid
43
+ def parse
44
+ translations = {}
45
+ idx = 0
46
+ idx = parse_line(idx, translations) while idx < @lines.length
47
+ translations
48
+ end
49
+
50
+ private
51
+
52
+ # Parses the entry that starts at idx, if any, and returns the next line
53
+ # to read.
54
+ def parse_line(idx, translations)
55
+ return idx + 1 unless @lines[idx].strip.start_with?('msgid ')
56
+
57
+ metadata = @mode == :simple ? {} : read_comments(idx)
58
+ parse_entry(idx, translations, metadata)
59
+ end
60
+
61
+ # --- Step 1: comment lines above msgid ---
62
+
63
+ # Reads the comment and blank lines above msgid_idx, nearest first. When a
64
+ # kind of comment appears more than once, the topmost one wins.
65
+ def read_comments(msgid_idx)
66
+ metadata = { reference: nil, fuzzy: false, previous_msgid: nil }
67
+ (msgid_idx - 1).downto(0) do |idx|
68
+ line = @lines[idx].strip
69
+ break unless line.empty? || line.start_with?('#')
70
+
71
+ read_comment(line, metadata)
72
+ end
73
+ @mode == :merge ? metadata : { reference: metadata[:reference] }
74
+ end
75
+
76
+ def read_comment(line, metadata)
77
+ if line.start_with?('#:')
78
+ metadata[:reference] = line.sub(/^#:\s*/, '').strip
79
+ elsif line.start_with?('#,') && line.include?('fuzzy')
80
+ metadata[:fuzzy] = true
81
+ elsif line.start_with?('#|') && line.include?('msgid')
82
+ metadata[:previous_msgid] = previous_msgid(line)
83
+ end
84
+ end
85
+
86
+ def previous_msgid(line)
87
+ raw = line.sub(/^#\|\s*msgid\s*/, '').strip
88
+ quote = raw[0]
89
+ return raw unless ['"', "'"].include?(quote) && raw.end_with?(quote)
90
+
91
+ unescape(raw[1...-1], quote)
92
+ end
93
+
94
+ # --- Step 2: msgid and msgstr fields ---
95
+
96
+ # Reads msgid and the msgstr right after it. A msgid with no msgstr on the
97
+ # next line is dropped, and that next line is skipped too.
98
+ def parse_entry(idx, translations, metadata)
99
+ msgid, idx = read_field(idx, MSGID_PATTERN)
100
+ return idx + 1 unless idx < @lines.length && @lines[idx].strip.start_with?('msgstr ')
101
+
102
+ msgstr, idx = read_field(idx, MSGSTR_PATTERN)
103
+ store(translations, msgid, msgstr, metadata)
104
+ idx
105
+ end
106
+
107
+ # @return [Array(String, Integer)] The field's value and the next line
108
+ def read_field(idx, pattern)
109
+ line = @lines[idx].strip
110
+ match = line.match(pattern)
111
+ quote = match ? line[match.begin(1) - 1] : '"'
112
+ values = match ? [unescape(match[1], quote)] : []
113
+ read_continuations(idx + 1, values, quote)
114
+ end
115
+
116
+ # Appends the quoted lines that follow a field to values.
117
+ #
118
+ # @return [Array(String, Integer)] The joined value and the next line
119
+ def read_continuations(idx, values, quote)
120
+ while idx < @lines.length && continuation?(@lines[idx].strip)
121
+ values << unescape(@lines[idx].strip[1...-1], quote)
122
+ idx += 1
123
+ end
124
+ [values.join, idx]
125
+ end
126
+
127
+ def continuation?(line)
128
+ return false if line.empty? || line.start_with?('#', 'msgid', 'msgstr')
129
+
130
+ (line.start_with?('"') && line.end_with?('"')) ||
131
+ (line.start_with?("'") && line.end_with?("'"))
132
+ end
133
+
134
+ # Unescapes a quoted PO value. Escaped quotes are replaced before escaped
135
+ # backslashes: the other order would read `\\"` as an escaped quote.
136
+ def unescape(str, quote)
137
+ str.gsub("\\#{quote}", quote).gsub('\\\\', '\\')
138
+ end
139
+
140
+ # --- Step 3: the stored value ---
141
+
142
+ def store(translations, msgid, msgstr, metadata)
143
+ return if msgid.empty?
144
+
145
+ translations[msgid] = @mode == :simple ? msgstr : entry(msgstr, metadata)
146
+ end
147
+
148
+ def entry(msgstr, metadata)
149
+ entry = { msgstr: msgstr }.merge(metadata.compact)
150
+ entry[:comment] = nil if @mode == :merge || metadata[:reference]
151
+ entry
152
+ end
153
+ end
154
+ end
155
+ end
@@ -26,92 +26,106 @@ module Jekyll
26
26
  #
27
27
  # Skips new_msgid values longer than MAX_FUZZY_MSGID_LENGTH (long strings are
28
28
  # unique HTML fragments with no useful near-duplicate and Levenshtein is O(n²)).
29
- # Pre-filters candidates to the length range where similarity ≥ threshold is
30
- # mathematically possible before invoking Levenshtein.
29
+ # Only candidates whose length allows a similarity ≥ threshold are scored, and
30
+ # each one stops being scored as soon as its edit distance is too large.
31
31
  #
32
32
  # @param new_msgid [String] the new source string to match
33
33
  # @param candidates [Hash] { old_msgid => entry } where entry is either a
34
34
  # String msgstr or a Hash with :msgstr key
35
35
  # @param threshold [Float] minimum similarity score to accept (0.0–1.0)
36
+ # @param scores [Hash, nil] cache of scores by [new_msgid, old_msgid], shared
37
+ # between calls that use the same threshold, such as the locales of one
38
+ # compendium merge. Without it, every call scores its candidates again.
36
39
  # @return [Hash, nil] { msgid: String, msgstr: String } or nil if no match
37
- def self.find_match(new_msgid, candidates, threshold: THRESHOLD)
40
+ def self.find_match(new_msgid, candidates, threshold: THRESHOLD, scores: nil)
38
41
  return nil if candidates.empty?
39
42
  return nil if new_msgid.nil? || new_msgid.length > Constants::MAX_FUZZY_MSGID_LENGTH
40
43
 
41
- len = new_msgid.length
42
- min_feas = (len * threshold).ceil
43
- max_feas = threshold.positive? ? (len / threshold).floor : Float::INFINITY
44
-
45
- best = best_candidate(new_msgid, candidates, min_feas, max_feas, threshold)
46
- return nil unless best
47
-
48
- { msgid: best[:msgid], msgstr: msgstr_from_entry(best[:entry]) }
44
+ best = best_candidate(new_msgid, candidates, threshold, scores)
45
+ best && { msgid: best[:msgid], msgstr: msgstr_from_entry(best[:entry]) }
49
46
  end
50
47
 
51
- def self.best_candidate(new_msgid, candidates, min_feas, max_feas, threshold)
52
- best_msgid = best_entry = nil
48
+ def self.best_candidate(new_msgid, candidates, threshold, scores)
49
+ lengths = feasible_lengths(new_msgid.length, threshold)
50
+ best = nil
53
51
  best_score = 0.0
54
52
 
55
53
  candidates.each do |old_msgid, entry|
56
- ol = old_msgid.length
57
- next if ol < min_feas || ol > max_feas
54
+ next unless lengths.cover?(old_msgid.length)
58
55
 
59
- score = similarity(new_msgid, old_msgid)
56
+ score = cached_score(new_msgid, old_msgid, threshold, scores)
60
57
  next unless score > best_score && score >= threshold
61
58
 
62
59
  best_score = score
63
- best_msgid = old_msgid
64
- best_entry = entry
60
+ best = { msgid: old_msgid, entry: entry }
65
61
  break if best_score >= 1.0
66
62
  end
63
+ best
64
+ end
67
65
 
68
- best_msgid ? { msgid: best_msgid, entry: best_entry } : nil
66
+ # Candidate lengths for which a similarity ≥ threshold is possible.
67
+ def self.feasible_lengths(length, threshold)
68
+ max = threshold.positive? ? (length / threshold).floor : Float::INFINITY
69
+ (length * threshold).ceil..max
69
70
  end
70
71
 
71
- # Normalized Levenshtein similarity between two strings.
72
- #
73
- # Returns 0.0 immediately when the length ratio falls below the threshold —
74
- # the maximum achievable similarity is min_len/max_len, so Levenshtein cannot
75
- # produce a useful result and the O(n²) computation is skipped.
76
- #
77
- # @param str_a [String]
78
- # @param str_b [String]
79
- # @return [Float] 0.0 (completely different) to 1.0 (identical)
80
- def self.similarity(str_a, str_b)
72
+ def self.cached_score(new_msgid, old_msgid, threshold, scores)
73
+ return bounded_similarity(new_msgid, old_msgid, threshold) unless scores
74
+
75
+ scores.fetch([new_msgid, old_msgid]) do |key|
76
+ scores[key] = bounded_similarity(new_msgid, old_msgid, threshold)
77
+ end
78
+ end
79
+
80
+ # Normalized Levenshtein similarity (0.0 to 1.0), or 0.0 as soon as the
81
+ # score is certain to be below threshold, without finishing the edit
82
+ # distance.
83
+ def self.bounded_similarity(str_a, str_b, threshold)
81
84
  return 1.0 if str_a == str_b
82
- return 0.0 if str_a.empty? || str_b.empty?
83
85
 
84
86
  max_len = [str_a.length, str_b.length].max
85
- min_len = [str_a.length, str_b.length].min
86
- return 0.0 if min_len.to_f / max_len < THRESHOLD
87
+ # Rounded up so a pair exactly at the threshold is never cut off.
88
+ limit = (max_len * (1 - threshold)).ceil
89
+ return 0.0 if char_count_distance(str_a, str_b) > limit
87
90
 
88
- dist = levenshtein(str_a, str_b)
89
- 1.0 - (dist.to_f / max_len)
91
+ distance = bounded_distance(str_a, str_b, limit)
92
+ distance > limit ? 0.0 : 1.0 - (distance.to_f / max_len)
90
93
  end
91
94
 
92
- def self.levenshtein(str_a, str_b)
93
- # Keep the shorter string as the row to minimise the dp array size.
94
- str_a, str_b = str_b, str_a if str_a.length > str_b.length
95
+ # A lower bound on the Levenshtein distance, from character counts alone:
96
+ # each edit changes the counts by at most one on each side.
97
+ def self.char_count_distance(str_a, str_b)
98
+ diff = str_a.each_char.tally
99
+ str_b.each_char { |char| diff[char] = diff.fetch(char, 0) - 1 }
100
+ surplus, deficit = diff.values.partition(&:positive?)
101
+ [surplus.sum, -deficit.sum].max
102
+ end
95
103
 
104
+ # Levenshtein distance, or any value above limit once every way to finish
105
+ # the comparison already costs more than limit.
106
+ def self.bounded_distance(str_a, str_b, limit)
107
+ # Keep the shorter string as the row to minimise the row size.
108
+ str_a, str_b = str_b, str_a if str_a.length > str_b.length
96
109
  a_chars = str_a.chars
97
- b_chars = str_b.chars
98
- m = a_chars.length
99
- n = b_chars.length
100
-
101
- curr = Array.new(m + 1) { |i| i }
102
- prev = Array.new(m + 1, 0)
103
-
104
- n.times do |j|
105
- curr, prev = prev, curr # swap in place — no allocation
106
- curr[0] = j + 1
107
- m.times do |i|
108
- cost = a_chars[i] == b_chars[j] ? 0 : 1
109
- curr[i + 1] = [curr[i] + 1, prev[i + 1] + 1, prev[i] + cost].min
110
- end
110
+ row = (0..a_chars.length).to_a
111
+
112
+ str_b.each_char.with_index(1) do |b_char, row_number|
113
+ row = next_row(row, a_chars, b_char, row_number)
114
+ return limit + 1 if row.min > limit
115
+ end
116
+ row.last
117
+ end
118
+
119
+ def self.next_row(prev, a_chars, b_char, row_number)
120
+ curr = [row_number]
121
+ a_chars.each_with_index do |a_char, i|
122
+ cost = a_char == b_char ? 0 : 1
123
+ curr << [curr[i] + 1, prev[i + 1] + 1, prev[i] + cost].min
111
124
  end
112
- curr[m]
125
+ curr
113
126
  end
114
- private_class_method :levenshtein, :best_candidate
127
+ private_class_method :best_candidate, :feasible_lengths, :cached_score,
128
+ :bounded_similarity, :char_count_distance, :bounded_distance, :next_row
115
129
  end
116
130
  end
117
131
  end
@@ -1,9 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative 'reader'
4
- require_relative 'manager'
5
4
  require_relative 'path_builder'
6
5
  require_relative '../utils/error_handler'
6
+ require_relative '../utils/msgstr_validator'
7
7
 
8
8
  module Jekyll
9
9
  module L10n
@@ -18,6 +18,7 @@ module Jekyll
18
18
  # * Construct file paths for locale/page combinations
19
19
  # * Load PO files from disk when not cached
20
20
  # * Parse PO file content into translation hashes
21
+ # * Drop entries whose msgstr markup does not match the msgid (MsgstrValidator)
21
22
  # * Cache parsed translations for reuse
22
23
  # * Handle file-not-found gracefully (return empty hash)
23
24
  # * Log errors during loading
@@ -26,11 +27,19 @@ module Jekyll
26
27
  # translations = PoFileLoader.load('_site', '_locales', 'es', 'docs/index.html')
27
28
  # # Returns cached or newly-loaded translations for that page/locale
28
29
  #
29
- # @see Jekyll::L10n::PoFileManager for caching implementation details
30
+ # @see Jekyll::L10n::PoFileManager for the public cache API
30
31
  class PoFileLoader
32
+ # The global PO file cache, mapping PO file paths to translation
33
+ # hashes. Read and clear it through PoFileManager.
34
+ #
35
+ # @return [Hash]
36
+ def self.cache
37
+ @cache ||= {}
38
+ end
39
+
31
40
  # Load a PO file with caching.
32
41
  #
33
- # First checks the global PoFileManager cache for already-loaded translations.
42
+ # First checks the global cache for already-loaded translations.
34
43
  # If not found, constructs the file path, loads from disk if file exists,
35
44
  # parses the PO file, caches the result, and returns. Returns empty hash
36
45
  # if file doesn't exist or if an error occurs during loading.
@@ -42,24 +51,24 @@ module Jekyll
42
51
  # If nil, loads compendium.
43
52
  # @return [Hash] Translation hash { msgid => msgstr } or empty hash if not found
44
53
  def self.load(source, locales_dir, locale, page_path)
45
- cache_key = "#{locale}:#{page_path}"
46
- cached = PoFileManager.cache[cache_key]
47
- return cached if cached
48
-
49
54
  po_path = PoPathBuilder.build(source, locales_dir, locale, page_path)
55
+ cached = cache[po_path]
56
+ return cached if cached
50
57
  return {} unless File.exist?(po_path)
51
58
 
52
- load_and_cache(cache_key, po_path)
59
+ load_and_cache(po_path)
53
60
  end
54
61
 
55
- def self.load_and_cache(cache_key, po_path)
56
- translations = PoFileReader.parse_for_translation(po_path)
57
- PoFileManager.cache[cache_key] = translations
62
+ def self.load_and_cache(po_path)
63
+ translations = MsgstrValidator.reject_invalid(PoFileReader.parse_for_translation(po_path), po_path)
64
+ cache[po_path] = translations
58
65
  translations
59
66
  rescue StandardError => e
60
67
  ErrorHandler.log_warning("loading PO file #{po_path}", e)
61
68
  {}
62
69
  end
70
+
71
+ private_class_method :load_and_cache
63
72
  end
64
73
  end
65
74
  end
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative '../constants'
3
4
  require_relative 'loader'
4
5
  require_relative 'writer'
5
6
  require_relative 'merger'
@@ -22,7 +23,7 @@ module Jekyll
22
23
  # * Load and cache PO files (page-specific and compendium)
23
24
  # * Save new or updated PO files with optional merging of existing translations
24
25
  # * Merge PO files for locales (combining page-specific with compendium)
25
- # * Manage global in-memory cache keyed by locale and page path
26
+ # * Manage global in-memory cache keyed by PO file path
26
27
  # * Clear cache when files are written or rebuilt
27
28
  # * Construct proper file paths for locale/page combinations
28
29
  #
@@ -31,25 +32,26 @@ module Jekyll
31
32
  # translations = manager.load_po_file('es', 'docs/index.html')
32
33
  # manager.save_po_file('es', entries, page_path: 'docs/index.html')
33
34
  class PoFileManager
34
- DEFAULT_LOCALES_DIR = '_locales'
35
+ DEFAULT_LOCALES_DIR = Constants::DEFAULT_LOCALES_DIR
35
36
  attr_reader :site, :locales_dir
36
37
 
37
38
  class << self
38
39
  # Access the global PO file cache.
39
40
  #
40
- # @return [Hash] Cache mapping "locale:page_path" keys to translation hashes
41
+ # @return [Hash] Cache mapping PO file paths to translation hashes
41
42
  def cache
42
- @cache ||= {}
43
+ PoFileLoader.cache
43
44
  end
44
45
 
45
46
  # Clear the global PO file cache.
46
47
  #
47
- # Clears all cached PO file translations. Should be called when the site
48
- # is rebuilt or PO files are modified to ensure fresh data is loaded.
48
+ # Clears all cached PO file translations. Runs on every site reset (see
49
+ # the hook at the end of this file), so each rebuild reloads PO files
50
+ # edited since the previous one.
49
51
  #
50
52
  # @return [void]
51
53
  def clear_cache
52
- @cache = {}
54
+ PoFileLoader.cache.clear
53
55
  end
54
56
 
55
57
  # Get the current size of the global cache.
@@ -64,7 +66,7 @@ module Jekyll
64
66
  #
65
67
  # @param site [Jekyll::Site] Jekyll site object
66
68
  # @param locales_dir [String] Directory containing PO files (defaults to
67
- # DEFAULT_LOCALES_DIR = "_locales")
69
+ # Constants::DEFAULT_LOCALES_DIR)
68
70
  def initialize(site, locales_dir = DEFAULT_LOCALES_DIR)
69
71
  @site = site
70
72
  @source = SiteConfigAccessor.source(@site)
@@ -98,6 +100,23 @@ module Jekyll
98
100
  PoFileLoader.load(@source, @locales_dir, locale, page_path)
99
101
  end
100
102
 
103
+ # Path of a locale's compendium in this manager's locales directory.
104
+ #
105
+ # @param locale [String] Target locale code (e.g., 'es', 'fr')
106
+ # @return [String] Full path to the compendium PO file
107
+ def compendium_path(locale)
108
+ PoPathBuilder.build(@source, @locales_dir, locale, nil)
109
+ end
110
+
111
+ # Directory holding a locale's page-specific PO files in this manager's
112
+ # locales directory.
113
+ #
114
+ # @param locale [String] Target locale code (e.g., 'es', 'fr')
115
+ # @return [String] Full path to the locale directory
116
+ def locale_dir(locale)
117
+ PoPathBuilder.locale_dir(@source, @locales_dir, locale)
118
+ end
119
+
101
120
  # Save a PO file for a locale and optional page.
102
121
  #
103
122
  # Writes entries to PO file, optionally merging with existing translations to
@@ -111,8 +130,7 @@ module Jekyll
111
130
  # @return [Boolean] True if write successful, false on error
112
131
  def save_po_file(locale, entries, page_path: nil, skip_merge: false)
113
132
  po_path = PoPathBuilder.build(@source, @locales_dir, locale, page_path)
114
- prepare_and_write_po_file(po_path, entries, locale, page_path: page_path,
115
- skip_merge: skip_merge)
133
+ prepare_and_write_po_file(po_path, entries, locale, skip_merge: skip_merge)
116
134
  rescue StandardError => e
117
135
  ErrorHandler.log_error("saving PO file #{po_path}", e)
118
136
  false
@@ -143,19 +161,16 @@ module Jekyll
143
161
 
144
162
  private
145
163
 
146
- def prepare_and_write_po_file(po_path, entries, locale, page_path: nil, skip_merge: false)
164
+ def prepare_and_write_po_file(po_path, entries, locale, skip_merge: false)
147
165
  LoggerFormatter.debug_if_enabled('PoFileManager',
148
166
  "Writing PO file: #{po_path} (#{entries.length} entries)")
149
167
  FileOperations.ensure_directory(po_path)
150
168
  PoFileWriter.write(po_path, entries, locale, skip_merge: skip_merge)
151
- invalidate_po_cache(locale, page_path)
169
+ self.class.cache.delete(po_path)
152
170
  true
153
171
  end
154
-
155
- def invalidate_po_cache(locale, page_path)
156
- cache_key = "#{locale}:#{page_path}"
157
- self.class.cache.delete(cache_key)
158
- end
159
172
  end
160
173
  end
161
174
  end
175
+
176
+ Jekyll::Hooks.register(:site, :after_reset) { Jekyll::L10n::PoFileManager.clear_cache }
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require_relative 'path_builder'
3
4
  require_relative 'reader'
4
5
  require_relative '../utils/error_handler'
5
6
 
@@ -37,7 +38,7 @@ module Jekyll
37
38
  # @return [Hash<String, Hash>] Merged translation hash where keys are msgid strings
38
39
  # and values are hashes with :msgstr and :reference keys
39
40
  def self.merge_for_locale(source, locales_dir, locale)
40
- locale_dir = File.join(source, locales_dir, locale)
41
+ locale_dir = PoPathBuilder.locale_dir(source, locales_dir, locale)
41
42
  return {} unless File.directory?(locale_dir)
42
43
 
43
44
  po_files = Dir.glob(File.join(locale_dir, '**', '*.po')).sort