jekyll-l10n 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +44 -0
- data/lib/jekyll-l10n/configuration/schema.rb +308 -0
- data/lib/jekyll-l10n/configuration.rb +165 -0
- data/lib/jekyll-l10n/constants.rb +45 -10
- data/lib/jekyll-l10n/errors.rb +7 -0
- data/lib/jekyll-l10n/extraction/compendium_merger.rb +9 -11
- data/lib/jekyll-l10n/extraction/compendium_translator.rb +27 -11
- data/lib/jekyll-l10n/extraction/config_loader.rb +4 -4
- data/lib/jekyll-l10n/extraction/coverage_report.rb +129 -0
- data/lib/jekyll-l10n/extraction/dom_text_extractor.rb +2 -31
- data/lib/jekyll-l10n/extraction/extractor.rb +48 -28
- data/lib/jekyll-l10n/extraction/html_string_extractor.rb +24 -10
- data/lib/jekyll-l10n/extraction/logger.rb +6 -5
- data/lib/jekyll-l10n/extraction/result_saver.rb +5 -10
- data/lib/jekyll-l10n/instrumentation.rb +172 -139
- data/lib/jekyll-l10n/jekyll/generator.rb +10 -22
- data/lib/jekyll-l10n/jekyll/localized_page.rb +27 -49
- data/lib/jekyll-l10n/jekyll/post_write_html_reprocessor.rb +42 -25
- data/lib/jekyll-l10n/jekyll/post_write_processor.rb +2 -1
- data/lib/jekyll-l10n/jekyll/regeneration_checker.rb +16 -10
- data/lib/jekyll-l10n/jekyll/url_filter.rb +8 -7
- data/lib/jekyll-l10n/po_file/entry_parser.rb +155 -0
- data/lib/jekyll-l10n/po_file/fuzzy_matcher.rb +67 -53
- data/lib/jekyll-l10n/po_file/loader.rb +20 -11
- data/lib/jekyll-l10n/po_file/manager.rb +32 -17
- data/lib/jekyll-l10n/po_file/merger.rb +2 -1
- data/lib/jekyll-l10n/po_file/path_builder.rb +11 -1
- data/lib/jekyll-l10n/po_file/reader.rb +40 -508
- data/lib/jekyll-l10n/po_file/writer.rb +31 -16
- data/lib/jekyll-l10n/translation/block_text_extractor.rb +3 -7
- data/lib/jekyll-l10n/translation/html_translator.rb +136 -157
- data/lib/jekyll-l10n/translation/libre_translate_languages.rb +73 -0
- data/lib/jekyll-l10n/translation/libre_translator.rb +134 -78
- data/lib/jekyll-l10n/translation/request_slots.rb +69 -0
- data/lib/jekyll-l10n/translation/translation_progress.rb +77 -0
- data/lib/jekyll-l10n/translation/translator.rb +20 -17
- data/lib/jekyll-l10n/utils/baseurl_normalizer.rb +108 -0
- data/lib/jekyll-l10n/utils/debug_logger.rb +4 -51
- data/lib/jekyll-l10n/utils/error_handler.rb +9 -24
- data/lib/jekyll-l10n/utils/external_link_icon_preserver.rb +5 -62
- data/lib/jekyll-l10n/utils/html_parser.rb +24 -31
- data/lib/jekyll-l10n/utils/html_text_utils.rb +146 -91
- data/lib/jekyll-l10n/utils/logger_formatter.rb +6 -4
- data/lib/jekyll-l10n/utils/msgstr_validator.rb +74 -0
- data/lib/jekyll-l10n/utils/page_locales_config.rb +126 -107
- data/lib/jekyll-l10n/utils/po_entry_converter.rb +0 -43
- data/lib/jekyll-l10n/utils/site_config_accessor.rb +56 -54
- data/lib/jekyll-l10n/utils/translation_resolver.rb +56 -6
- data/lib/jekyll-l10n/utils/url_path_builder.rb +5 -27
- data/lib/jekyll-l10n/utils/url_transformer.rb +5 -11
- data/lib/jekyll-l10n/version.rb +1 -1
- data/lib/jekyll-l10n.rb +9 -6
- metadata +12 -3
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative '../configuration'
|
|
4
|
+
require_relative '../constants'
|
|
3
5
|
require_relative '../utils/site_config_accessor'
|
|
4
6
|
require_relative '../po_file/path_builder'
|
|
5
7
|
require_relative '../utils/locale_url_formatter'
|
|
@@ -30,13 +32,10 @@ module Jekyll
|
|
|
30
32
|
# end
|
|
31
33
|
#
|
|
32
34
|
class RegenerationChecker
|
|
33
|
-
# @!visibility private
|
|
34
|
-
DEFAULT_LOCALES_DIR = '_locales'
|
|
35
|
-
|
|
36
35
|
# @param site [Jekyll::Site] The Jekyll site
|
|
37
36
|
# @param locales_dir [String] Directory, relative to the site source, that
|
|
38
|
-
# holds the PO files. Defaults to DEFAULT_LOCALES_DIR.
|
|
39
|
-
def initialize(site, locales_dir = DEFAULT_LOCALES_DIR)
|
|
37
|
+
# holds the PO files. Defaults to Constants::DEFAULT_LOCALES_DIR.
|
|
38
|
+
def initialize(site, locales_dir = Constants::DEFAULT_LOCALES_DIR)
|
|
40
39
|
@site = site
|
|
41
40
|
@locales_dir = locales_dir
|
|
42
41
|
@source = SiteConfigAccessor.source(@site)
|
|
@@ -79,12 +78,17 @@ module Jekyll
|
|
|
79
78
|
#
|
|
80
79
|
# This must produce the same path as LocalizedPage#destination.
|
|
81
80
|
#
|
|
81
|
+
# It is the English page's own output path, as Jekyll computes it, under
|
|
82
|
+
# the locale segment, since the localized page's URL is the English URL
|
|
83
|
+
# with that segment in front (LocalizedPage#url).
|
|
84
|
+
#
|
|
82
85
|
# @param page [Jekyll::Page] The source (English) page
|
|
83
86
|
# @param locale [String] Locale code, e.g. 'es' or 'pt_BR'
|
|
84
|
-
# @return [String] Path such as "es/about/index.html"
|
|
87
|
+
# @return [String] Path such as "es/about/index.html" or "es/about.html"
|
|
88
|
+
# (no leading slash)
|
|
85
89
|
def relative_dest_path(page, locale)
|
|
86
|
-
|
|
87
|
-
"#{
|
|
90
|
+
output_path = page.destination(@destination).delete_prefix(@destination)
|
|
91
|
+
"#{LocaleUrlFormatter.to_url_segment(locale)}#{output_path}"
|
|
88
92
|
end
|
|
89
93
|
|
|
90
94
|
private
|
|
@@ -92,11 +96,13 @@ module Jekyll
|
|
|
92
96
|
def incremental_enabled?
|
|
93
97
|
return false if @site&.config.nil?
|
|
94
98
|
|
|
95
|
-
@site
|
|
99
|
+
SiteConfigAccessor.site_settings(@site)['incremental'] == true
|
|
96
100
|
end
|
|
97
101
|
|
|
102
|
+
# A directory at the path is the layout of an earlier release
|
|
103
|
+
# (es/nav.html/index.html), so the page is written again.
|
|
98
104
|
def dest_path_exists?(page, locale)
|
|
99
|
-
File.
|
|
105
|
+
File.file?(dest_path(page, locale))
|
|
100
106
|
end
|
|
101
107
|
|
|
102
108
|
def dest_path(page, locale)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'liquid'
|
|
4
|
+
require_relative '../configuration'
|
|
4
5
|
require_relative '../constants'
|
|
5
6
|
require_relative '../utils/error_handler'
|
|
6
7
|
require_relative '../utils/locale_name_resolver'
|
|
@@ -27,7 +28,7 @@ module Jekyll
|
|
|
27
28
|
# @example Usage in Jekyll Liquid templates
|
|
28
29
|
# <!-- Add locale prefix to URL -->
|
|
29
30
|
# <a href="{{ '/about/' | locale_url: 'es_ES' }}">Acerca de</a>
|
|
30
|
-
# <!-- Produces: /es-
|
|
31
|
+
# <!-- Produces: /es-es/about/ -->
|
|
31
32
|
#
|
|
32
33
|
# <!-- Switch to different locale variant -->
|
|
33
34
|
# <a href="{{ page.url | switch_locale_url: 'fr' }}">Français</a>
|
|
@@ -41,7 +42,7 @@ module Jekyll
|
|
|
41
42
|
# Generate a locale-prefixed URL for the given URL and locale
|
|
42
43
|
#
|
|
43
44
|
# Adds a locale prefix in BCP 47 hyphenated form to the given URL. For example,
|
|
44
|
-
# '/about/' becomes '/es-
|
|
45
|
+
# '/about/' becomes '/es-es/about/' for locale 'es_ES'. The filter handles
|
|
45
46
|
# various edge cases and only applies the prefix when appropriate.
|
|
46
47
|
#
|
|
47
48
|
# The method performs several checks to determine if prefixing is appropriate:
|
|
@@ -56,7 +57,7 @@ module Jekyll
|
|
|
56
57
|
# prefixing is not appropriate
|
|
57
58
|
# @example
|
|
58
59
|
# <!-- Explicit underscore locale converted to hyphen in URL -->
|
|
59
|
-
# {{ '/about/' | locale_url: 'es_ES' }} # => '/es-
|
|
60
|
+
# {{ '/about/' | locale_url: 'es_ES' }} # => '/es-es/about/'
|
|
60
61
|
#
|
|
61
62
|
# <!-- Current page's locale -->
|
|
62
63
|
# {{ '/about/' | locale_url }} # => '/es/about/' (if page locale is 'es')
|
|
@@ -71,7 +72,7 @@ module Jekyll
|
|
|
71
72
|
# Generate a URL for the given locale variant of the current page
|
|
72
73
|
#
|
|
73
74
|
# Switches the current URL to a different locale variant. For example, if the current
|
|
74
|
-
# page is '/es-
|
|
75
|
+
# page is '/es-es/about/', calling this with 'fr' returns '/fr/about/'. This is useful
|
|
75
76
|
# for building language switcher menus.
|
|
76
77
|
#
|
|
77
78
|
# This filter first extracts the base URL (without locale prefix) from the given URL,
|
|
@@ -81,7 +82,7 @@ module Jekyll
|
|
|
81
82
|
# @param target_locale [String] The target locale code (e.g., 'es_ES', 'fr', 'pt_BR')
|
|
82
83
|
# @return [String] The URL for the target locale, or original URL if locale is invalid
|
|
83
84
|
# @example
|
|
84
|
-
# <!-- Current page is /es-
|
|
85
|
+
# <!-- Current page is /es-es/about/, switch to French -->
|
|
85
86
|
# {{ page.url | switch_locale_url: 'fr' }} # => '/fr/about/'
|
|
86
87
|
#
|
|
87
88
|
# <!-- Build a language switcher -->
|
|
@@ -117,7 +118,7 @@ module Jekyll
|
|
|
117
118
|
# @example
|
|
118
119
|
# {{ page.url | locale_menu }}
|
|
119
120
|
# # => [{"code"=>"en","name"=>"English","url"=>"/about/","current"=>false},
|
|
120
|
-
# # {"code"=>"es_ES","name"=>"Spanish (Spain)","url"=>"/es-
|
|
121
|
+
# # {"code"=>"es_ES","name"=>"Spanish (Spain)","url"=>"/es-es/about/","current"=>true}]
|
|
121
122
|
def locale_menu(url)
|
|
122
123
|
menu_locales.map do |locale|
|
|
123
124
|
{
|
|
@@ -207,7 +208,7 @@ module Jekyll
|
|
|
207
208
|
|
|
208
209
|
if page
|
|
209
210
|
locales = if page.is_a?(Hash)
|
|
210
|
-
|
|
211
|
+
Configuration::Schema.normalize(page[Configuration::PAGE_DATA_KEY], source: nil)['locales']
|
|
211
212
|
else
|
|
212
213
|
SiteConfigAccessor.page_locales_data(page)['locales']
|
|
213
214
|
end
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Jekyll
|
|
4
|
+
module L10n
|
|
5
|
+
# Parses the text of a GNU Gettext PO file into a translation hash.
|
|
6
|
+
#
|
|
7
|
+
# Each entry is read in three steps, in this order:
|
|
8
|
+
# 1. The comment lines above `msgid`: reference (`#:`), fuzzy flag (`#,`)
|
|
9
|
+
# and previous msgid (`#|`).
|
|
10
|
+
# 2. The `msgid` and `msgstr` fields, each with its continuation lines.
|
|
11
|
+
# 3. The stored value, whose shape depends on the mode.
|
|
12
|
+
#
|
|
13
|
+
# Modes and the value stored for each msgid:
|
|
14
|
+
# * :simple — the msgstr String
|
|
15
|
+
# * :references — { msgstr:, reference: } (reference only when present)
|
|
16
|
+
# * :merge — { msgstr:, reference:, fuzzy:, previous_msgid: }
|
|
17
|
+
#
|
|
18
|
+
# Entries with an empty msgid (the header) are skipped.
|
|
19
|
+
#
|
|
20
|
+
# @example
|
|
21
|
+
# PoEntryParser.parse(%(msgid "Hello"\nmsgstr "Hola"\n), :simple)
|
|
22
|
+
# # => { "Hello" => "Hola" }
|
|
23
|
+
class PoEntryParser
|
|
24
|
+
MODES = %i[simple references merge].freeze
|
|
25
|
+
MSGID_PATTERN = /^msgid ['"](.*)['"] *$/.freeze
|
|
26
|
+
MSGSTR_PATTERN = /^msgstr ['"](.*)['"] *$/.freeze
|
|
27
|
+
|
|
28
|
+
# @param content [String] PO file text
|
|
29
|
+
# @param mode [Symbol] One of MODES
|
|
30
|
+
# @return [Hash] Translations keyed by msgid
|
|
31
|
+
def self.parse(content, mode)
|
|
32
|
+
raise ArgumentError, "unknown PO parse mode: #{mode.inspect}" unless MODES.include?(mode)
|
|
33
|
+
|
|
34
|
+
new(content.to_s.split("\n"), mode).parse
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def initialize(lines, mode)
|
|
38
|
+
@lines = lines
|
|
39
|
+
@mode = mode
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# @return [Hash] Translations keyed by msgid
|
|
43
|
+
def parse
|
|
44
|
+
translations = {}
|
|
45
|
+
idx = 0
|
|
46
|
+
idx = parse_line(idx, translations) while idx < @lines.length
|
|
47
|
+
translations
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
# Parses the entry that starts at idx, if any, and returns the next line
|
|
53
|
+
# to read.
|
|
54
|
+
def parse_line(idx, translations)
|
|
55
|
+
return idx + 1 unless @lines[idx].strip.start_with?('msgid ')
|
|
56
|
+
|
|
57
|
+
metadata = @mode == :simple ? {} : read_comments(idx)
|
|
58
|
+
parse_entry(idx, translations, metadata)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# --- Step 1: comment lines above msgid ---
|
|
62
|
+
|
|
63
|
+
# Reads the comment and blank lines above msgid_idx, nearest first. When a
|
|
64
|
+
# kind of comment appears more than once, the topmost one wins.
|
|
65
|
+
def read_comments(msgid_idx)
|
|
66
|
+
metadata = { reference: nil, fuzzy: false, previous_msgid: nil }
|
|
67
|
+
(msgid_idx - 1).downto(0) do |idx|
|
|
68
|
+
line = @lines[idx].strip
|
|
69
|
+
break unless line.empty? || line.start_with?('#')
|
|
70
|
+
|
|
71
|
+
read_comment(line, metadata)
|
|
72
|
+
end
|
|
73
|
+
@mode == :merge ? metadata : { reference: metadata[:reference] }
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def read_comment(line, metadata)
|
|
77
|
+
if line.start_with?('#:')
|
|
78
|
+
metadata[:reference] = line.sub(/^#:\s*/, '').strip
|
|
79
|
+
elsif line.start_with?('#,') && line.include?('fuzzy')
|
|
80
|
+
metadata[:fuzzy] = true
|
|
81
|
+
elsif line.start_with?('#|') && line.include?('msgid')
|
|
82
|
+
metadata[:previous_msgid] = previous_msgid(line)
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def previous_msgid(line)
|
|
87
|
+
raw = line.sub(/^#\|\s*msgid\s*/, '').strip
|
|
88
|
+
quote = raw[0]
|
|
89
|
+
return raw unless ['"', "'"].include?(quote) && raw.end_with?(quote)
|
|
90
|
+
|
|
91
|
+
unescape(raw[1...-1], quote)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# --- Step 2: msgid and msgstr fields ---
|
|
95
|
+
|
|
96
|
+
# Reads msgid and the msgstr right after it. A msgid with no msgstr on the
|
|
97
|
+
# next line is dropped, and that next line is skipped too.
|
|
98
|
+
def parse_entry(idx, translations, metadata)
|
|
99
|
+
msgid, idx = read_field(idx, MSGID_PATTERN)
|
|
100
|
+
return idx + 1 unless idx < @lines.length && @lines[idx].strip.start_with?('msgstr ')
|
|
101
|
+
|
|
102
|
+
msgstr, idx = read_field(idx, MSGSTR_PATTERN)
|
|
103
|
+
store(translations, msgid, msgstr, metadata)
|
|
104
|
+
idx
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# @return [Array(String, Integer)] The field's value and the next line
|
|
108
|
+
def read_field(idx, pattern)
|
|
109
|
+
line = @lines[idx].strip
|
|
110
|
+
match = line.match(pattern)
|
|
111
|
+
quote = match ? line[match.begin(1) - 1] : '"'
|
|
112
|
+
values = match ? [unescape(match[1], quote)] : []
|
|
113
|
+
read_continuations(idx + 1, values, quote)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Appends the quoted lines that follow a field to values.
|
|
117
|
+
#
|
|
118
|
+
# @return [Array(String, Integer)] The joined value and the next line
|
|
119
|
+
def read_continuations(idx, values, quote)
|
|
120
|
+
while idx < @lines.length && continuation?(@lines[idx].strip)
|
|
121
|
+
values << unescape(@lines[idx].strip[1...-1], quote)
|
|
122
|
+
idx += 1
|
|
123
|
+
end
|
|
124
|
+
[values.join, idx]
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def continuation?(line)
|
|
128
|
+
return false if line.empty? || line.start_with?('#', 'msgid', 'msgstr')
|
|
129
|
+
|
|
130
|
+
(line.start_with?('"') && line.end_with?('"')) ||
|
|
131
|
+
(line.start_with?("'") && line.end_with?("'"))
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Unescapes a quoted PO value. Escaped quotes are replaced before escaped
|
|
135
|
+
# backslashes: the other order would read `\\"` as an escaped quote.
|
|
136
|
+
def unescape(str, quote)
|
|
137
|
+
str.gsub("\\#{quote}", quote).gsub('\\\\', '\\')
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# --- Step 3: the stored value ---
|
|
141
|
+
|
|
142
|
+
def store(translations, msgid, msgstr, metadata)
|
|
143
|
+
return if msgid.empty?
|
|
144
|
+
|
|
145
|
+
translations[msgid] = @mode == :simple ? msgstr : entry(msgstr, metadata)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def entry(msgstr, metadata)
|
|
149
|
+
entry = { msgstr: msgstr }.merge(metadata.compact)
|
|
150
|
+
entry[:comment] = nil if @mode == :merge || metadata[:reference]
|
|
151
|
+
entry
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
end
|
|
@@ -26,92 +26,106 @@ module Jekyll
|
|
|
26
26
|
#
|
|
27
27
|
# Skips new_msgid values longer than MAX_FUZZY_MSGID_LENGTH (long strings are
|
|
28
28
|
# unique HTML fragments with no useful near-duplicate and Levenshtein is O(n²)).
|
|
29
|
-
#
|
|
30
|
-
#
|
|
29
|
+
# Only candidates whose length allows a similarity ≥ threshold are scored, and
|
|
30
|
+
# each one stops being scored as soon as its edit distance is too large.
|
|
31
31
|
#
|
|
32
32
|
# @param new_msgid [String] the new source string to match
|
|
33
33
|
# @param candidates [Hash] { old_msgid => entry } where entry is either a
|
|
34
34
|
# String msgstr or a Hash with :msgstr key
|
|
35
35
|
# @param threshold [Float] minimum similarity score to accept (0.0–1.0)
|
|
36
|
+
# @param scores [Hash, nil] cache of scores by [new_msgid, old_msgid], shared
|
|
37
|
+
# between calls that use the same threshold, such as the locales of one
|
|
38
|
+
# compendium merge. Without it, every call scores its candidates again.
|
|
36
39
|
# @return [Hash, nil] { msgid: String, msgstr: String } or nil if no match
|
|
37
|
-
def self.find_match(new_msgid, candidates, threshold: THRESHOLD)
|
|
40
|
+
def self.find_match(new_msgid, candidates, threshold: THRESHOLD, scores: nil)
|
|
38
41
|
return nil if candidates.empty?
|
|
39
42
|
return nil if new_msgid.nil? || new_msgid.length > Constants::MAX_FUZZY_MSGID_LENGTH
|
|
40
43
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
max_feas = threshold.positive? ? (len / threshold).floor : Float::INFINITY
|
|
44
|
-
|
|
45
|
-
best = best_candidate(new_msgid, candidates, min_feas, max_feas, threshold)
|
|
46
|
-
return nil unless best
|
|
47
|
-
|
|
48
|
-
{ msgid: best[:msgid], msgstr: msgstr_from_entry(best[:entry]) }
|
|
44
|
+
best = best_candidate(new_msgid, candidates, threshold, scores)
|
|
45
|
+
best && { msgid: best[:msgid], msgstr: msgstr_from_entry(best[:entry]) }
|
|
49
46
|
end
|
|
50
47
|
|
|
51
|
-
def self.best_candidate(new_msgid, candidates,
|
|
52
|
-
|
|
48
|
+
def self.best_candidate(new_msgid, candidates, threshold, scores)
|
|
49
|
+
lengths = feasible_lengths(new_msgid.length, threshold)
|
|
50
|
+
best = nil
|
|
53
51
|
best_score = 0.0
|
|
54
52
|
|
|
55
53
|
candidates.each do |old_msgid, entry|
|
|
56
|
-
|
|
57
|
-
next if ol < min_feas || ol > max_feas
|
|
54
|
+
next unless lengths.cover?(old_msgid.length)
|
|
58
55
|
|
|
59
|
-
score =
|
|
56
|
+
score = cached_score(new_msgid, old_msgid, threshold, scores)
|
|
60
57
|
next unless score > best_score && score >= threshold
|
|
61
58
|
|
|
62
59
|
best_score = score
|
|
63
|
-
|
|
64
|
-
best_entry = entry
|
|
60
|
+
best = { msgid: old_msgid, entry: entry }
|
|
65
61
|
break if best_score >= 1.0
|
|
66
62
|
end
|
|
63
|
+
best
|
|
64
|
+
end
|
|
67
65
|
|
|
68
|
-
|
|
66
|
+
# Candidate lengths for which a similarity ≥ threshold is possible.
|
|
67
|
+
def self.feasible_lengths(length, threshold)
|
|
68
|
+
max = threshold.positive? ? (length / threshold).floor : Float::INFINITY
|
|
69
|
+
(length * threshold).ceil..max
|
|
69
70
|
end
|
|
70
71
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
#
|
|
80
|
-
|
|
72
|
+
def self.cached_score(new_msgid, old_msgid, threshold, scores)
|
|
73
|
+
return bounded_similarity(new_msgid, old_msgid, threshold) unless scores
|
|
74
|
+
|
|
75
|
+
scores.fetch([new_msgid, old_msgid]) do |key|
|
|
76
|
+
scores[key] = bounded_similarity(new_msgid, old_msgid, threshold)
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Normalized Levenshtein similarity (0.0 to 1.0), or 0.0 as soon as the
|
|
81
|
+
# score is certain to be below threshold, without finishing the edit
|
|
82
|
+
# distance.
|
|
83
|
+
def self.bounded_similarity(str_a, str_b, threshold)
|
|
81
84
|
return 1.0 if str_a == str_b
|
|
82
|
-
return 0.0 if str_a.empty? || str_b.empty?
|
|
83
85
|
|
|
84
86
|
max_len = [str_a.length, str_b.length].max
|
|
85
|
-
|
|
86
|
-
|
|
87
|
+
# Rounded up so a pair exactly at the threshold is never cut off.
|
|
88
|
+
limit = (max_len * (1 - threshold)).ceil
|
|
89
|
+
return 0.0 if char_count_distance(str_a, str_b) > limit
|
|
87
90
|
|
|
88
|
-
|
|
89
|
-
1.0 - (
|
|
91
|
+
distance = bounded_distance(str_a, str_b, limit)
|
|
92
|
+
distance > limit ? 0.0 : 1.0 - (distance.to_f / max_len)
|
|
90
93
|
end
|
|
91
94
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
+
# A lower bound on the Levenshtein distance, from character counts alone:
|
|
96
|
+
# each edit changes the counts by at most one on each side.
|
|
97
|
+
def self.char_count_distance(str_a, str_b)
|
|
98
|
+
diff = str_a.each_char.tally
|
|
99
|
+
str_b.each_char { |char| diff[char] = diff.fetch(char, 0) - 1 }
|
|
100
|
+
surplus, deficit = diff.values.partition(&:positive?)
|
|
101
|
+
[surplus.sum, -deficit.sum].max
|
|
102
|
+
end
|
|
95
103
|
|
|
104
|
+
# Levenshtein distance, or any value above limit once every way to finish
|
|
105
|
+
# the comparison already costs more than limit.
|
|
106
|
+
def self.bounded_distance(str_a, str_b, limit)
|
|
107
|
+
# Keep the shorter string as the row to minimise the row size.
|
|
108
|
+
str_a, str_b = str_b, str_a if str_a.length > str_b.length
|
|
96
109
|
a_chars = str_a.chars
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
110
|
+
row = (0..a_chars.length).to_a
|
|
111
|
+
|
|
112
|
+
str_b.each_char.with_index(1) do |b_char, row_number|
|
|
113
|
+
row = next_row(row, a_chars, b_char, row_number)
|
|
114
|
+
return limit + 1 if row.min > limit
|
|
115
|
+
end
|
|
116
|
+
row.last
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def self.next_row(prev, a_chars, b_char, row_number)
|
|
120
|
+
curr = [row_number]
|
|
121
|
+
a_chars.each_with_index do |a_char, i|
|
|
122
|
+
cost = a_char == b_char ? 0 : 1
|
|
123
|
+
curr << [curr[i] + 1, prev[i + 1] + 1, prev[i] + cost].min
|
|
111
124
|
end
|
|
112
|
-
curr
|
|
125
|
+
curr
|
|
113
126
|
end
|
|
114
|
-
private_class_method :
|
|
127
|
+
private_class_method :best_candidate, :feasible_lengths, :cached_score,
|
|
128
|
+
:bounded_similarity, :char_count_distance, :bounded_distance, :next_row
|
|
115
129
|
end
|
|
116
130
|
end
|
|
117
131
|
end
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative 'reader'
|
|
4
|
-
require_relative 'manager'
|
|
5
4
|
require_relative 'path_builder'
|
|
6
5
|
require_relative '../utils/error_handler'
|
|
6
|
+
require_relative '../utils/msgstr_validator'
|
|
7
7
|
|
|
8
8
|
module Jekyll
|
|
9
9
|
module L10n
|
|
@@ -18,6 +18,7 @@ module Jekyll
|
|
|
18
18
|
# * Construct file paths for locale/page combinations
|
|
19
19
|
# * Load PO files from disk when not cached
|
|
20
20
|
# * Parse PO file content into translation hashes
|
|
21
|
+
# * Drop entries whose msgstr markup does not match the msgid (MsgstrValidator)
|
|
21
22
|
# * Cache parsed translations for reuse
|
|
22
23
|
# * Handle file-not-found gracefully (return empty hash)
|
|
23
24
|
# * Log errors during loading
|
|
@@ -26,11 +27,19 @@ module Jekyll
|
|
|
26
27
|
# translations = PoFileLoader.load('_site', '_locales', 'es', 'docs/index.html')
|
|
27
28
|
# # Returns cached or newly-loaded translations for that page/locale
|
|
28
29
|
#
|
|
29
|
-
# @see Jekyll::L10n::PoFileManager for
|
|
30
|
+
# @see Jekyll::L10n::PoFileManager for the public cache API
|
|
30
31
|
class PoFileLoader
|
|
32
|
+
# The global PO file cache, mapping PO file paths to translation
|
|
33
|
+
# hashes. Read and clear it through PoFileManager.
|
|
34
|
+
#
|
|
35
|
+
# @return [Hash]
|
|
36
|
+
def self.cache
|
|
37
|
+
@cache ||= {}
|
|
38
|
+
end
|
|
39
|
+
|
|
31
40
|
# Load a PO file with caching.
|
|
32
41
|
#
|
|
33
|
-
# First checks the global
|
|
42
|
+
# First checks the global cache for already-loaded translations.
|
|
34
43
|
# If not found, constructs the file path, loads from disk if file exists,
|
|
35
44
|
# parses the PO file, caches the result, and returns. Returns empty hash
|
|
36
45
|
# if file doesn't exist or if an error occurs during loading.
|
|
@@ -42,24 +51,24 @@ module Jekyll
|
|
|
42
51
|
# If nil, loads compendium.
|
|
43
52
|
# @return [Hash] Translation hash { msgid => msgstr } or empty hash if not found
|
|
44
53
|
def self.load(source, locales_dir, locale, page_path)
|
|
45
|
-
cache_key = "#{locale}:#{page_path}"
|
|
46
|
-
cached = PoFileManager.cache[cache_key]
|
|
47
|
-
return cached if cached
|
|
48
|
-
|
|
49
54
|
po_path = PoPathBuilder.build(source, locales_dir, locale, page_path)
|
|
55
|
+
cached = cache[po_path]
|
|
56
|
+
return cached if cached
|
|
50
57
|
return {} unless File.exist?(po_path)
|
|
51
58
|
|
|
52
|
-
load_and_cache(
|
|
59
|
+
load_and_cache(po_path)
|
|
53
60
|
end
|
|
54
61
|
|
|
55
|
-
def self.load_and_cache(
|
|
56
|
-
translations = PoFileReader.parse_for_translation(po_path)
|
|
57
|
-
|
|
62
|
+
def self.load_and_cache(po_path)
|
|
63
|
+
translations = MsgstrValidator.reject_invalid(PoFileReader.parse_for_translation(po_path), po_path)
|
|
64
|
+
cache[po_path] = translations
|
|
58
65
|
translations
|
|
59
66
|
rescue StandardError => e
|
|
60
67
|
ErrorHandler.log_warning("loading PO file #{po_path}", e)
|
|
61
68
|
{}
|
|
62
69
|
end
|
|
70
|
+
|
|
71
|
+
private_class_method :load_and_cache
|
|
63
72
|
end
|
|
64
73
|
end
|
|
65
74
|
end
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative '../constants'
|
|
3
4
|
require_relative 'loader'
|
|
4
5
|
require_relative 'writer'
|
|
5
6
|
require_relative 'merger'
|
|
@@ -22,7 +23,7 @@ module Jekyll
|
|
|
22
23
|
# * Load and cache PO files (page-specific and compendium)
|
|
23
24
|
# * Save new or updated PO files with optional merging of existing translations
|
|
24
25
|
# * Merge PO files for locales (combining page-specific with compendium)
|
|
25
|
-
# * Manage global in-memory cache keyed by
|
|
26
|
+
# * Manage global in-memory cache keyed by PO file path
|
|
26
27
|
# * Clear cache when files are written or rebuilt
|
|
27
28
|
# * Construct proper file paths for locale/page combinations
|
|
28
29
|
#
|
|
@@ -31,25 +32,26 @@ module Jekyll
|
|
|
31
32
|
# translations = manager.load_po_file('es', 'docs/index.html')
|
|
32
33
|
# manager.save_po_file('es', entries, page_path: 'docs/index.html')
|
|
33
34
|
class PoFileManager
|
|
34
|
-
DEFAULT_LOCALES_DIR =
|
|
35
|
+
DEFAULT_LOCALES_DIR = Constants::DEFAULT_LOCALES_DIR
|
|
35
36
|
attr_reader :site, :locales_dir
|
|
36
37
|
|
|
37
38
|
class << self
|
|
38
39
|
# Access the global PO file cache.
|
|
39
40
|
#
|
|
40
|
-
# @return [Hash] Cache mapping
|
|
41
|
+
# @return [Hash] Cache mapping PO file paths to translation hashes
|
|
41
42
|
def cache
|
|
42
|
-
|
|
43
|
+
PoFileLoader.cache
|
|
43
44
|
end
|
|
44
45
|
|
|
45
46
|
# Clear the global PO file cache.
|
|
46
47
|
#
|
|
47
|
-
# Clears all cached PO file translations.
|
|
48
|
-
#
|
|
48
|
+
# Clears all cached PO file translations. Runs on every site reset (see
|
|
49
|
+
# the hook at the end of this file), so each rebuild reloads PO files
|
|
50
|
+
# edited since the previous one.
|
|
49
51
|
#
|
|
50
52
|
# @return [void]
|
|
51
53
|
def clear_cache
|
|
52
|
-
|
|
54
|
+
PoFileLoader.cache.clear
|
|
53
55
|
end
|
|
54
56
|
|
|
55
57
|
# Get the current size of the global cache.
|
|
@@ -64,7 +66,7 @@ module Jekyll
|
|
|
64
66
|
#
|
|
65
67
|
# @param site [Jekyll::Site] Jekyll site object
|
|
66
68
|
# @param locales_dir [String] Directory containing PO files (defaults to
|
|
67
|
-
# DEFAULT_LOCALES_DIR
|
|
69
|
+
# Constants::DEFAULT_LOCALES_DIR)
|
|
68
70
|
def initialize(site, locales_dir = DEFAULT_LOCALES_DIR)
|
|
69
71
|
@site = site
|
|
70
72
|
@source = SiteConfigAccessor.source(@site)
|
|
@@ -98,6 +100,23 @@ module Jekyll
|
|
|
98
100
|
PoFileLoader.load(@source, @locales_dir, locale, page_path)
|
|
99
101
|
end
|
|
100
102
|
|
|
103
|
+
# Path of a locale's compendium in this manager's locales directory.
|
|
104
|
+
#
|
|
105
|
+
# @param locale [String] Target locale code (e.g., 'es', 'fr')
|
|
106
|
+
# @return [String] Full path to the compendium PO file
|
|
107
|
+
def compendium_path(locale)
|
|
108
|
+
PoPathBuilder.build(@source, @locales_dir, locale, nil)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Directory holding a locale's page-specific PO files in this manager's
|
|
112
|
+
# locales directory.
|
|
113
|
+
#
|
|
114
|
+
# @param locale [String] Target locale code (e.g., 'es', 'fr')
|
|
115
|
+
# @return [String] Full path to the locale directory
|
|
116
|
+
def locale_dir(locale)
|
|
117
|
+
PoPathBuilder.locale_dir(@source, @locales_dir, locale)
|
|
118
|
+
end
|
|
119
|
+
|
|
101
120
|
# Save a PO file for a locale and optional page.
|
|
102
121
|
#
|
|
103
122
|
# Writes entries to PO file, optionally merging with existing translations to
|
|
@@ -111,8 +130,7 @@ module Jekyll
|
|
|
111
130
|
# @return [Boolean] True if write successful, false on error
|
|
112
131
|
def save_po_file(locale, entries, page_path: nil, skip_merge: false)
|
|
113
132
|
po_path = PoPathBuilder.build(@source, @locales_dir, locale, page_path)
|
|
114
|
-
prepare_and_write_po_file(po_path, entries, locale,
|
|
115
|
-
skip_merge: skip_merge)
|
|
133
|
+
prepare_and_write_po_file(po_path, entries, locale, skip_merge: skip_merge)
|
|
116
134
|
rescue StandardError => e
|
|
117
135
|
ErrorHandler.log_error("saving PO file #{po_path}", e)
|
|
118
136
|
false
|
|
@@ -143,19 +161,16 @@ module Jekyll
|
|
|
143
161
|
|
|
144
162
|
private
|
|
145
163
|
|
|
146
|
-
def prepare_and_write_po_file(po_path, entries, locale,
|
|
164
|
+
def prepare_and_write_po_file(po_path, entries, locale, skip_merge: false)
|
|
147
165
|
LoggerFormatter.debug_if_enabled('PoFileManager',
|
|
148
166
|
"Writing PO file: #{po_path} (#{entries.length} entries)")
|
|
149
167
|
FileOperations.ensure_directory(po_path)
|
|
150
168
|
PoFileWriter.write(po_path, entries, locale, skip_merge: skip_merge)
|
|
151
|
-
|
|
169
|
+
self.class.cache.delete(po_path)
|
|
152
170
|
true
|
|
153
171
|
end
|
|
154
|
-
|
|
155
|
-
def invalidate_po_cache(locale, page_path)
|
|
156
|
-
cache_key = "#{locale}:#{page_path}"
|
|
157
|
-
self.class.cache.delete(cache_key)
|
|
158
|
-
end
|
|
159
172
|
end
|
|
160
173
|
end
|
|
161
174
|
end
|
|
175
|
+
|
|
176
|
+
Jekyll::Hooks.register(:site, :after_reset) { Jekyll::L10n::PoFileManager.clear_cache }
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative 'path_builder'
|
|
3
4
|
require_relative 'reader'
|
|
4
5
|
require_relative '../utils/error_handler'
|
|
5
6
|
|
|
@@ -37,7 +38,7 @@ module Jekyll
|
|
|
37
38
|
# @return [Hash<String, Hash>] Merged translation hash where keys are msgid strings
|
|
38
39
|
# and values are hashes with :msgstr and :reference keys
|
|
39
40
|
def self.merge_for_locale(source, locales_dir, locale)
|
|
40
|
-
locale_dir =
|
|
41
|
+
locale_dir = PoPathBuilder.locale_dir(source, locales_dir, locale)
|
|
41
42
|
return {} unless File.directory?(locale_dir)
|
|
42
43
|
|
|
43
44
|
po_files = Dir.glob(File.join(locale_dir, '**', '*.po')).sort
|