jekyll-l10n 1.7.1 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +44 -0
- data/lib/jekyll-l10n/configuration/schema.rb +308 -0
- data/lib/jekyll-l10n/configuration.rb +165 -0
- data/lib/jekyll-l10n/constants.rb +45 -10
- data/lib/jekyll-l10n/errors.rb +7 -0
- data/lib/jekyll-l10n/extraction/compendium_merger.rb +9 -11
- data/lib/jekyll-l10n/extraction/compendium_translator.rb +27 -11
- data/lib/jekyll-l10n/extraction/config_loader.rb +28 -58
- data/lib/jekyll-l10n/extraction/coverage_report.rb +129 -0
- data/lib/jekyll-l10n/extraction/dom_text_extractor.rb +2 -31
- data/lib/jekyll-l10n/extraction/extractor.rb +72 -65
- data/lib/jekyll-l10n/extraction/html_string_extractor.rb +24 -10
- data/lib/jekyll-l10n/extraction/logger.rb +6 -5
- data/lib/jekyll-l10n/extraction/result_saver.rb +5 -10
- data/lib/jekyll-l10n/instrumentation.rb +172 -139
- data/lib/jekyll-l10n/jekyll/generator.rb +47 -61
- data/lib/jekyll-l10n/jekyll/localized_page.rb +27 -49
- data/lib/jekyll-l10n/jekyll/post_write_html_reprocessor.rb +53 -48
- data/lib/jekyll-l10n/jekyll/post_write_processor.rb +2 -1
- data/lib/jekyll-l10n/jekyll/regeneration_checker.rb +70 -50
- data/lib/jekyll-l10n/jekyll/url_filter.rb +13 -13
- data/lib/jekyll-l10n/po_file/entry_parser.rb +155 -0
- data/lib/jekyll-l10n/po_file/fuzzy_matcher.rb +67 -53
- data/lib/jekyll-l10n/po_file/loader.rb +20 -11
- data/lib/jekyll-l10n/po_file/manager.rb +32 -17
- data/lib/jekyll-l10n/po_file/merger.rb +2 -1
- data/lib/jekyll-l10n/po_file/path_builder.rb +11 -1
- data/lib/jekyll-l10n/po_file/reader.rb +40 -508
- data/lib/jekyll-l10n/po_file/writer.rb +31 -16
- data/lib/jekyll-l10n/translation/block_text_extractor.rb +3 -7
- data/lib/jekyll-l10n/translation/html_translator.rb +136 -157
- data/lib/jekyll-l10n/translation/libre_translate_languages.rb +73 -0
- data/lib/jekyll-l10n/translation/libre_translator.rb +134 -78
- data/lib/jekyll-l10n/translation/page_translation_loader.rb +13 -29
- data/lib/jekyll-l10n/translation/request_slots.rb +69 -0
- data/lib/jekyll-l10n/translation/translation_progress.rb +77 -0
- data/lib/jekyll-l10n/translation/translator.rb +49 -78
- data/lib/jekyll-l10n/utils/baseurl_normalizer.rb +108 -0
- data/lib/jekyll-l10n/utils/debug_logger.rb +4 -51
- data/lib/jekyll-l10n/utils/error_handler.rb +9 -24
- data/lib/jekyll-l10n/utils/external_link_icon_preserver.rb +5 -62
- data/lib/jekyll-l10n/utils/html_parser.rb +24 -31
- data/lib/jekyll-l10n/utils/html_text_utils.rb +146 -91
- data/lib/jekyll-l10n/utils/logger_formatter.rb +6 -4
- data/lib/jekyll-l10n/utils/msgstr_validator.rb +74 -0
- data/lib/jekyll-l10n/utils/page_locales_config.rb +165 -223
- data/lib/jekyll-l10n/utils/po_entry_converter.rb +0 -43
- data/lib/jekyll-l10n/utils/site_config_accessor.rb +82 -42
- data/lib/jekyll-l10n/utils/translation_resolver.rb +56 -6
- data/lib/jekyll-l10n/utils/url_path_builder.rb +5 -27
- data/lib/jekyll-l10n/utils/url_transformer.rb +5 -11
- data/lib/jekyll-l10n/version.rb +1 -1
- data/lib/jekyll-l10n.rb +67 -176
- metadata +12 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d6469f52a4c1ac2cc94d6e6c546a1091b91d1d6b30331110931496ad96422404
|
|
4
|
+
data.tar.gz: f9726ad1727591b5f2aebce4435f8dd99ad9e9269d6fd669a0a402c156e4219e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 62f17f5dcf12c508b5f2998d93d30b443aa3107a289cda6472b6df9a3249016516c4b5f6c309e5e258e20469d7bc6de957aefdbdd49575e80db0ddb325481578
|
|
7
|
+
data.tar.gz: dc55f0983289039c97b0c968d295685616ab467aeb766ea41729777629ac70c3c09a9585cd14dd3208dce59d64fd700a031107b71ba91a293f0fd102709a0920
|
data/README.md
CHANGED
|
@@ -6,11 +6,55 @@ translatable strings from your site's HTML, organizes them into PO files for
|
|
|
6
6
|
professional translation, and automatically applies translations to generate
|
|
7
7
|
fully localized pages with locale-prefixed URLs.
|
|
8
8
|
|
|
9
|
+
## Installation
|
|
10
|
+
|
|
11
|
+
Add the gem to your site's `Gemfile` and the plugin to `_config.yml`:
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
bundle add jekyll-l10n --group jekyll_plugins
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
```yaml
|
|
18
|
+
plugins:
|
|
19
|
+
- jekyll-l10n
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Then follow [Getting started](https://centos.gitlab.io/artwork/centos-web/jekyll-l10n/getting-started/)
|
|
23
|
+
to enable localization.
|
|
24
|
+
|
|
9
25
|
## Documentation
|
|
10
26
|
|
|
11
27
|
For complete documentation, configuration options, and examples, visit the
|
|
12
28
|
[documentation site](https://centos.gitlab.io/artwork/centos-web/jekyll-l10n/).
|
|
13
29
|
|
|
30
|
+
## Contributing
|
|
31
|
+
|
|
32
|
+
1. Install the development dependencies:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
bundle install
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
2. Run the tests and lint, the same checks CI runs before a release:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
make # runs both of the commands below
|
|
42
|
+
bundle exec rspec
|
|
43
|
+
bundle exec rubocop
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
`bundle exec rspec` also writes a coverage report to `coverage/`.
|
|
47
|
+
|
|
48
|
+
3. Write commit messages in the
|
|
49
|
+
[Conventional Commits](https://www.conventionalcommits.org/) format, for
|
|
50
|
+
example `fix(translation): keep inline links in block translations`.
|
|
51
|
+
Releases and the changelog are generated from them: `feat` and `fix`
|
|
52
|
+
commits, and commits marked as breaking changes, publish a new version.
|
|
53
|
+
|
|
54
|
+
4. If an AI tool helped write a commit, add an `Assisted-by:` trailer, as the
|
|
55
|
+
[Fedora AI contribution policy](https://docs.fedoraproject.org/en-US/council/policy/ai-contribution-policy/)
|
|
56
|
+
requires.
|
|
57
|
+
|
|
14
58
|
## License
|
|
15
59
|
|
|
16
60
|
MIT License - see [LICENSE](LICENSE) file for details.
|
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'did_you_mean'
|
|
4
|
+
require_relative '../constants'
|
|
5
|
+
|
|
6
|
+
module Jekyll
|
|
7
|
+
module L10n
|
|
8
|
+
module Configuration
|
|
9
|
+
# The type, scope and default of every setting, and how a value of the
|
|
10
|
+
# wrong type is converted or dropped, with a warning.
|
|
11
|
+
#
|
|
12
|
+
# {.normalize} cleans a settings hash: it drops unknown keys, converts
|
|
13
|
+
# near-misses such as "false", and drops values it cannot read so the
|
|
14
|
+
# default applies. It never adds defaults, so a setting the user did not
|
|
15
|
+
# set stays unset; PageLocalesConfig applies the defaults when asked.
|
|
16
|
+
# That matters for LibreTranslate, which turns on only when the user set
|
|
17
|
+
# `libretranslate_api_url` themselves.
|
|
18
|
+
#
|
|
19
|
+
# A bad locale code is dropped here, with a warning, so the page is built
|
|
20
|
+
# for its other locales. The LibreTranslate URL, format and source locale
|
|
21
|
+
# are checked by PageLocalesConfig instead, which turns machine
|
|
22
|
+
# translation off, with a warning, when one is invalid.
|
|
23
|
+
module Schema
|
|
24
|
+
# Every setting, by dotted path: its type, its scope, and its default
|
|
25
|
+
# (nil: none, or worked out by PageLocalesConfig). A :page setting is
|
|
26
|
+
# set under `with_locales_data` (page front matter or `_config.yml`
|
|
27
|
+
# defaults); a :site setting applies to the whole build and is set
|
|
28
|
+
# under the top-level `jekyll_l10n` key.
|
|
29
|
+
#
|
|
30
|
+
# @return [Hash{String => Hash}]
|
|
31
|
+
SETTINGS = {
|
|
32
|
+
'incremental' => { type: :boolean, scope: :site, default: Constants::DEFAULT_INCREMENTAL },
|
|
33
|
+
'locales' => { type: :locale_list, scope: :page, default: [] },
|
|
34
|
+
'locales_dir' => { type: :string, scope: :page, default: Constants::DEFAULT_LOCALES_DIR },
|
|
35
|
+
'extract_on_build' => { type: :boolean, scope: :page, default: Constants::DEFAULT_EXTRACT_ON_BUILD },
|
|
36
|
+
'update_compendium' => { type: :boolean, scope: :site, default: Constants::DEFAULT_UPDATE_COMPENDIUM },
|
|
37
|
+
'logging.show_statistics' => { type: :boolean, scope: :site, default: Constants::DEFAULT_SHOW_STATISTICS },
|
|
38
|
+
'logging.debug' => { type: :boolean, scope: :page, default: Constants::DEFAULT_DEBUG_LOGGING },
|
|
39
|
+
'logging.trace' => { type: :boolean, scope: :site, default: Constants::DEFAULT_TRACE_LOGGING },
|
|
40
|
+
'translation.fallback' => { type: :fallback, scope: :page, default: Constants::DEFAULT_FALLBACK_MODE },
|
|
41
|
+
'translation.min_coverage' => { type: :percentage, scope: :site, default: nil },
|
|
42
|
+
'translation.libretranslate_enabled' => { type: :boolean, scope: :site, default: nil },
|
|
43
|
+
'translation.libretranslate_api_url' => {
|
|
44
|
+
type: :string, scope: :site, default: Constants::DEFAULT_LIBRETRANSLATE_API_URL
|
|
45
|
+
},
|
|
46
|
+
'translation.libretranslate_api_key' => { type: :string, scope: :site, default: nil, secret: true },
|
|
47
|
+
'translation.libretranslate_source_locale' => {
|
|
48
|
+
type: :string, scope: :site, default: Constants::DEFAULT_LIBRETRANSLATE_SOURCE_LOCALE
|
|
49
|
+
},
|
|
50
|
+
'translation.libretranslate_format' => {
|
|
51
|
+
type: :string, scope: :site, default: Constants::DEFAULT_LIBRETRANSLATE_FORMAT
|
|
52
|
+
},
|
|
53
|
+
'translation.libretranslate_timeout' => {
|
|
54
|
+
type: :positive_integer, scope: :site, default: Constants::DEFAULT_LIBRETRANSLATE_TIMEOUT
|
|
55
|
+
},
|
|
56
|
+
'translation.libretranslate_batch_size' => {
|
|
57
|
+
type: :positive_integer, scope: :site, default: Constants::DEFAULT_LIBRETRANSLATE_BATCH_SIZE
|
|
58
|
+
},
|
|
59
|
+
'translation.libretranslate_concurrency' => {
|
|
60
|
+
type: :positive_integer, scope: :site, default: Constants::DEFAULT_LIBRETRANSLATE_CONCURRENCY
|
|
61
|
+
},
|
|
62
|
+
'translation.libretranslate_retry_attempts' => {
|
|
63
|
+
type: :non_negative_integer, scope: :site, default: Constants::DEFAULT_LIBRETRANSLATE_RETRY_ATTEMPTS
|
|
64
|
+
},
|
|
65
|
+
'translation.libretranslate_retry_delay' => {
|
|
66
|
+
type: :non_negative_integer, scope: :site, default: Constants::DEFAULT_LIBRETRANSLATE_RETRY_DELAY
|
|
67
|
+
},
|
|
68
|
+
'translation.libretranslate_stop_on_error' => {
|
|
69
|
+
type: :boolean, scope: :site, default: Constants::DEFAULT_LIBRETRANSLATE_STOP_ON_ERROR
|
|
70
|
+
},
|
|
71
|
+
'translation.libretranslate_mark_fuzzy' => {
|
|
72
|
+
type: :boolean, scope: :site, default: Constants::DEFAULT_LIBRETRANSLATE_MARK_FUZZY
|
|
73
|
+
},
|
|
74
|
+
'translation.libretranslate_progress_interval' => {
|
|
75
|
+
type: :non_negative_integer, scope: :site, default: Constants::DEFAULT_LIBRETRANSLATE_PROGRESS_INTERVAL
|
|
76
|
+
},
|
|
77
|
+
'extraction.translatable_attributes' => {
|
|
78
|
+
type: :string_list, scope: :page, default: Constants::DEFAULT_TRANSLATABLE_ATTRIBUTES
|
|
79
|
+
},
|
|
80
|
+
'extraction.exclude_selectors' => {
|
|
81
|
+
type: :string_list, scope: :page, default: Constants::DEFAULT_EXCLUDE_SELECTORS
|
|
82
|
+
}
|
|
83
|
+
}.freeze
|
|
84
|
+
|
|
85
|
+
# Whether value is a locale code such as "es" or "pt_BR".
|
|
86
|
+
LOCALE_CODE = ->(value) { value.is_a?(String) && Constants::LOCALE_PATTERN.match?(value) }
|
|
87
|
+
|
|
88
|
+
# Converts "true"/"false" in any case, surrounding spaces ignored.
|
|
89
|
+
BOOLEAN_WORD = lambda { |value|
|
|
90
|
+
word = value.strip.downcase if value.is_a?(String)
|
|
91
|
+
[word == 'true'] if %w[true false].include?(word)
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
# Converts a string of digits to an Integer.
|
|
95
|
+
INTEGER_STRING = ->(value) { [value.strip.to_i] if value.is_a?(String) && value.strip.match?(/\A\d+\z/) }
|
|
96
|
+
|
|
97
|
+
# Per type: what the warning says to expect, which values pass as is,
|
|
98
|
+
# and how a near-miss converts (returns [converted], or nil when the
|
|
99
|
+
# value can't be converted and the default applies instead).
|
|
100
|
+
#
|
|
101
|
+
# @return [Hash{Symbol => Hash}]
|
|
102
|
+
COERCERS = {
|
|
103
|
+
boolean: {
|
|
104
|
+
expected: 'true or false',
|
|
105
|
+
accept: ->(value) { [true, false].include?(value) },
|
|
106
|
+
convert: BOOLEAN_WORD
|
|
107
|
+
},
|
|
108
|
+
string: {
|
|
109
|
+
expected: 'a string',
|
|
110
|
+
# Empty strings pass: an empty libretranslate_api_url makes
|
|
111
|
+
# PageLocalesConfig turn machine translation off, with a warning.
|
|
112
|
+
accept: ->(value) { value.is_a?(String) },
|
|
113
|
+
convert: ->(_value) {}
|
|
114
|
+
},
|
|
115
|
+
positive_integer: {
|
|
116
|
+
expected: 'a positive integer',
|
|
117
|
+
accept: ->(value) { value.is_a?(Integer) && value.positive? },
|
|
118
|
+
convert: ->(value) { INTEGER_STRING.call(value)&.then { |(int)| [int] if int.positive? } }
|
|
119
|
+
},
|
|
120
|
+
non_negative_integer: {
|
|
121
|
+
expected: 'an integer of 0 or more',
|
|
122
|
+
accept: ->(value) { value.is_a?(Integer) && !value.negative? },
|
|
123
|
+
convert: INTEGER_STRING
|
|
124
|
+
},
|
|
125
|
+
percentage: {
|
|
126
|
+
expected: 'an integer from 0 to 100',
|
|
127
|
+
accept: ->(value) { value.is_a?(Integer) && value.between?(0, 100) },
|
|
128
|
+
convert: ->(value) { INTEGER_STRING.call(value)&.then { |(int)| [int] if int <= 100 } }
|
|
129
|
+
},
|
|
130
|
+
string_list: {
|
|
131
|
+
expected: 'a list of strings',
|
|
132
|
+
accept: ->(value) { value.is_a?(Array) && value.all?(String) },
|
|
133
|
+
convert: ->(value) { [[value]] if value.is_a?(String) && !value.strip.empty? }
|
|
134
|
+
},
|
|
135
|
+
# A single code reads as a one-code list. Codes that don't match
|
|
136
|
+
# LOCALE_PATTERN are dropped, so the page skips only those locales.
|
|
137
|
+
locale_list: {
|
|
138
|
+
expected: 'a list of locale codes such as es or pt_BR',
|
|
139
|
+
accept: ->(value) { value.is_a?(Array) && value.all?(&LOCALE_CODE) },
|
|
140
|
+
convert: lambda { |value|
|
|
141
|
+
codes = value.is_a?(String) ? [value] : value
|
|
142
|
+
[codes.select(&LOCALE_CODE)] if codes.is_a?(Array)
|
|
143
|
+
}
|
|
144
|
+
},
|
|
145
|
+
fallback: {
|
|
146
|
+
expected: "one of #{Constants::FALLBACK_MODES.join(', ')}",
|
|
147
|
+
accept: ->(value) { Constants::FALLBACK_MODES.include?(value) },
|
|
148
|
+
convert: lambda { |value|
|
|
149
|
+
mode = value.to_s.strip.downcase if value.is_a?(String) || value.is_a?(Symbol)
|
|
150
|
+
[mode] if Constants::FALLBACK_MODES.include?(mode)
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}.freeze
|
|
154
|
+
|
|
155
|
+
# The scope of the settings under each root key. PageLocalesConfig
|
|
156
|
+
# reads :all, the per-page and site-wide settings merged.
|
|
157
|
+
#
|
|
158
|
+
# @return [Hash{String => Symbol}]
|
|
159
|
+
ROOT_SCOPES = { PAGE_DATA_KEY => :page, CONFIG_KEY => :site }.freeze
|
|
160
|
+
|
|
161
|
+
# Every section name and setting name per scope (:page, :site, :all),
|
|
162
|
+
# by parent path ('' for the top level). Used to find unknown keys and
|
|
163
|
+
# suggest known ones.
|
|
164
|
+
#
|
|
165
|
+
# @return [Hash{Symbol => Hash{String => Array<String>}}]
|
|
166
|
+
KNOWN_KEYS = %i[page site all].to_h do |scope|
|
|
167
|
+
paths = SETTINGS.select { |_path, spec| scope == :all || spec[:scope] == scope }.keys
|
|
168
|
+
known = paths.each_with_object(Hash.new { |h, k| h[k] = [] }) do |path, keys|
|
|
169
|
+
*sections, name = path.split('.')
|
|
170
|
+
keys[sections.join('.')] |= [name]
|
|
171
|
+
keys[''] |= [sections.first] unless sections.empty?
|
|
172
|
+
end
|
|
173
|
+
[scope, known.to_h.freeze]
|
|
174
|
+
end.freeze
|
|
175
|
+
|
|
176
|
+
# Returns a cleaned copy of a settings hash: unknown keys and settings
|
|
177
|
+
# of the other scope dropped, near-misses converted, unreadable values
|
|
178
|
+
# dropped. Warns once per problem (see Configuration.warn_once).
|
|
179
|
+
#
|
|
180
|
+
# @param settings [Object] The settings, normally a Hash
|
|
181
|
+
# @param source [String, nil] Where the settings come from, for
|
|
182
|
+
# warnings; nil cleans without warning (for values already checked)
|
|
183
|
+
# @param root [String] The key the settings sit under, for warnings
|
|
184
|
+
# @param scope [Symbol] The settings to keep: :page, :site or :all.
|
|
185
|
+
# Defaults to the scope of root (see ROOT_SCOPES).
|
|
186
|
+
# @return [Hash]
|
|
187
|
+
def self.normalize(settings, source:, root: PAGE_DATA_KEY, scope: ROOT_SCOPES.fetch(root, :all))
|
|
188
|
+
return {} if settings.nil?
|
|
189
|
+
return normalize_section(settings, '', source, root, scope) if settings.is_a?(Hash)
|
|
190
|
+
|
|
191
|
+
warn_invalid(source, root, settings, 'a mapping of settings', 'ignoring it')
|
|
192
|
+
{}
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# Reads a true/false flag such as `with_locales`. nil is false; "true"
|
|
196
|
+
# and "false" convert with a warning; anything else is false, with a
|
|
197
|
+
# warning.
|
|
198
|
+
#
|
|
199
|
+
# @param value [Object] The flag's value
|
|
200
|
+
# @param key [String] The flag's name, for warnings
|
|
201
|
+
# @param source [String] Where the value comes from, for warnings
|
|
202
|
+
# @return [Boolean]
|
|
203
|
+
def self.boolean_flag(value, key, source)
|
|
204
|
+
return value if [true, false].include?(value)
|
|
205
|
+
return false if value.nil?
|
|
206
|
+
|
|
207
|
+
converted = BOOLEAN_WORD.call(value)
|
|
208
|
+
if converted
|
|
209
|
+
warn_invalid(source, key, value, 'true or false', "reading it as #{converted.first}")
|
|
210
|
+
return converted.first
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
warn_invalid(source, key, value, 'true or false', 'using false')
|
|
214
|
+
false
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
# Logs "<source>: <key path> should be <expected>, got <value>;
|
|
218
|
+
# <outcome>." once per build. Secret values are never shown.
|
|
219
|
+
#
|
|
220
|
+
# @return [void]
|
|
221
|
+
def self.warn_invalid(source, key_path, value, expected, outcome)
|
|
222
|
+
return if source.nil?
|
|
223
|
+
|
|
224
|
+
shown = SETTINGS.dig(key_path.split('.', 2).last, :secret) ? '[hidden]' : value.inspect
|
|
225
|
+
Configuration.warn_once("#{source}: #{key_path} should be #{expected}, got #{shown}; #{outcome}.")
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
def self.normalize_section(settings, parent, source, root, scope)
|
|
229
|
+
settings.each_with_object({}) do |(key, value), clean|
|
|
230
|
+
next if value.nil?
|
|
231
|
+
|
|
232
|
+
path = parent.empty? ? key.to_s : "#{parent}.#{key}"
|
|
233
|
+
cleaned = normalize_value(path, value, source, root, scope)
|
|
234
|
+
clean[key] = cleaned unless cleaned.nil?
|
|
235
|
+
end
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
# The cleaned value of the setting or section at path, or nil to leave
|
|
239
|
+
# it out.
|
|
240
|
+
def self.normalize_value(path, value, source, root, scope)
|
|
241
|
+
spec = SETTINGS[path]
|
|
242
|
+
# A copy, so freezing the result never freezes the site config.
|
|
243
|
+
return coerce(value, path, source, root)&.dup if spec && [:all, spec[:scope]].include?(scope)
|
|
244
|
+
return warn_wrong_scope(source, root, path, spec[:scope]) if spec
|
|
245
|
+
# A section of the other scope is still walked, so each of its
|
|
246
|
+
# settings gets a warning that names where it belongs.
|
|
247
|
+
return normalize_subsection(value, path, source, root, scope) if KNOWN_KEYS[:all][''].include?(path)
|
|
248
|
+
|
|
249
|
+
parent, _, key = path.rpartition('.')
|
|
250
|
+
warn_unknown(source, root, parent, key, scope)
|
|
251
|
+
nil
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
def self.normalize_subsection(value, path, source, root, scope)
|
|
255
|
+
return normalize_section(value, path, source, root, scope) if value.is_a?(Hash)
|
|
256
|
+
|
|
257
|
+
warn_invalid(source, "#{root}.#{path}", value, 'a mapping of settings', 'ignoring it')
|
|
258
|
+
nil
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
# Returns value if its type accepts it; otherwise its conversion, with
|
|
262
|
+
# a warning; otherwise nil (the default applies), with a warning.
|
|
263
|
+
def self.coerce(value, path, source, root)
|
|
264
|
+
spec = SETTINGS.fetch(path)
|
|
265
|
+
coercer = COERCERS.fetch(spec[:type])
|
|
266
|
+
return value if coercer[:accept].call(value)
|
|
267
|
+
|
|
268
|
+
key_path = "#{root}.#{path}"
|
|
269
|
+
converted = coercer[:convert].call(value)
|
|
270
|
+
if converted
|
|
271
|
+
outcome = spec[:secret] ? 'reading it as text' : "reading it as #{converted.first.inspect}"
|
|
272
|
+
warn_invalid(source, key_path, value, coercer[:expected], outcome)
|
|
273
|
+
return converted.first
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
warn_invalid(source, key_path, value, coercer[:expected], 'using the default')
|
|
277
|
+
nil
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
# Warns that a setting sits under the wrong root key, and returns nil
|
|
281
|
+
# so the setting is left out.
|
|
282
|
+
def self.warn_wrong_scope(source, root, path, setting_scope)
|
|
283
|
+
return if source.nil?
|
|
284
|
+
|
|
285
|
+
hint = if setting_scope == :site
|
|
286
|
+
"it applies to the whole build; set `#{CONFIG_KEY}.#{path}` in _config.yml"
|
|
287
|
+
else
|
|
288
|
+
"it applies per page; set `#{PAGE_DATA_KEY}.#{path}` in _config.yml defaults or page front matter"
|
|
289
|
+
end
|
|
290
|
+
Configuration.warn_once("#{source}: #{root}.#{path} is ignored here, #{hint}.")
|
|
291
|
+
nil
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
def self.warn_unknown(source, root, parent, key, scope)
|
|
295
|
+
return if source.nil?
|
|
296
|
+
|
|
297
|
+
key_path = [root, parent, key].reject { |part| part.to_s.empty? }.join('.')
|
|
298
|
+
dictionary = KNOWN_KEYS[scope].fetch(parent, [])
|
|
299
|
+
suggestion = DidYouMean::SpellChecker.new(dictionary: dictionary).correct(key.to_s).first
|
|
300
|
+
hint = suggestion ? "; did you mean `#{suggestion}`?" : '.'
|
|
301
|
+
Configuration.warn_once("#{source}: #{key_path} is not a known setting and is ignored#{hint}")
|
|
302
|
+
end
|
|
303
|
+
private_class_method :normalize_section, :normalize_value, :normalize_subsection, :coerce,
|
|
304
|
+
:warn_wrong_scope, :warn_unknown
|
|
305
|
+
end
|
|
306
|
+
end
|
|
307
|
+
end
|
|
308
|
+
end
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'jekyll'
|
|
4
|
+
require 'set'
|
|
5
|
+
|
|
6
|
+
module Jekyll
|
|
7
|
+
module L10n
|
|
8
|
+
# Names the configuration keys the plugin reads, checks their values, and
|
|
9
|
+
# resolves each page's settings once per build.
|
|
10
|
+
#
|
|
11
|
+
# Localization is configured per page: a page opts in with `with_locales`
|
|
12
|
+
# and reads its settings from `with_locales_data`, its front matter merged
|
|
13
|
+
# over the matching `_config.yml` defaults. Settings that apply to the
|
|
14
|
+
# whole build live under the top-level `jekyll_l10n` key of `_config.yml`
|
|
15
|
+
# (see {.site}); Schema gives each setting's scope.
|
|
16
|
+
#
|
|
17
|
+
# Values are checked where they are written, so a warning names its
|
|
18
|
+
# source: each `defaults` entry once per build, and each page's own front
|
|
19
|
+
# matter once. Schema describes every setting.
|
|
20
|
+
module Configuration
|
|
21
|
+
# The top-level _config.yml key for build-wide settings.
|
|
22
|
+
# @return [String]
|
|
23
|
+
CONFIG_KEY = 'jekyll_l10n'
|
|
24
|
+
|
|
25
|
+
# The page key that opts a page into localization.
|
|
26
|
+
# @return [String]
|
|
27
|
+
PAGE_FLAG_KEY = 'with_locales'
|
|
28
|
+
|
|
29
|
+
# The page key that holds a page's localization settings.
|
|
30
|
+
# @return [String]
|
|
31
|
+
PAGE_DATA_KEY = 'with_locales_data'
|
|
32
|
+
|
|
33
|
+
# Source name used in warnings about the merged `_config.yml` defaults.
|
|
34
|
+
# @return [String]
|
|
35
|
+
DEFAULTS_SOURCE = '_config.yml defaults'
|
|
36
|
+
|
|
37
|
+
# Source name used in warnings about the `jekyll_l10n` settings.
|
|
38
|
+
# @return [String]
|
|
39
|
+
SITE_SOURCE = '_config.yml'
|
|
40
|
+
|
|
41
|
+
# Whether the page opted into localization. "true" and "false" are read
|
|
42
|
+
# as booleans, with a warning; any other non-boolean value is false.
|
|
43
|
+
#
|
|
44
|
+
# @param page [Jekyll::Page] The page
|
|
45
|
+
# @return [Boolean]
|
|
46
|
+
def self.localizable?(page)
|
|
47
|
+
own = page.data.respond_to?(:key?) && page.data.key?(PAGE_FLAG_KEY)
|
|
48
|
+
Schema.boolean_flag(page.data[PAGE_FLAG_KEY], PAGE_FLAG_KEY, own ? source_of(page) : DEFAULTS_SOURCE)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# The page's checked `with_locales_data`: its own front matter merged
|
|
52
|
+
# over the `_config.yml` defaults that match the page. Nested hashes
|
|
53
|
+
# merge; lists such as `locales` are replaced. Frozen, and worked out
|
|
54
|
+
# once per page per build.
|
|
55
|
+
#
|
|
56
|
+
# Jekyll's `page.data` fills in defaults one top-level key at a time, so
|
|
57
|
+
# as soon as a page sets any `with_locales_data` key, `page.data` loses
|
|
58
|
+
# every site default for it. Read page settings through this method.
|
|
59
|
+
#
|
|
60
|
+
# @param page [Jekyll::Page] The page. Any other object gets only its own value.
|
|
61
|
+
# @return [Hash] The page's settings
|
|
62
|
+
def self.page(page)
|
|
63
|
+
pages = cache[:pages] ||= {}.compare_by_identity
|
|
64
|
+
pages[page] ||= deep_freeze(resolve_page(page))
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# The checked top-level `jekyll_l10n` settings, which apply to the whole
|
|
68
|
+
# build. A per-page setting found there is dropped with a warning.
|
|
69
|
+
# Frozen, and worked out once per build.
|
|
70
|
+
#
|
|
71
|
+
# @param site_config [Hash] The site configuration (site.config)
|
|
72
|
+
# @return [Hash] The settings, or {} when `jekyll_l10n` is not set
|
|
73
|
+
def self.site(site_config)
|
|
74
|
+
sites = cache[:sites] ||= {}.compare_by_identity
|
|
75
|
+
sites[site_config] ||= deep_freeze(
|
|
76
|
+
Schema.normalize(site_config[CONFIG_KEY], source: SITE_SOURCE, root: CONFIG_KEY)
|
|
77
|
+
)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Checks every `defaults` entry's `with_locales_data` and `with_locales`
|
|
81
|
+
# once per build, so each warning names the entry (`defaults[2]`).
|
|
82
|
+
#
|
|
83
|
+
# @param site_config [Hash] The site configuration (site.config)
|
|
84
|
+
# @return [void]
|
|
85
|
+
def self.check_defaults(site_config)
|
|
86
|
+
checked = cache[:checked_defaults] ||= Set.new.compare_by_identity
|
|
87
|
+
return unless checked.add?(site_config)
|
|
88
|
+
|
|
89
|
+
Array(site_config['defaults']).each_with_index do |entry, index|
|
|
90
|
+
values = entry['values'] if entry.is_a?(Hash)
|
|
91
|
+
check_defaults_values(values, "_config.yml defaults[#{index}]") if values.is_a?(Hash)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Logs a warning unless the same message was logged earlier in this
|
|
96
|
+
# build, so a bad value in a `defaults` entry warns once, not per page.
|
|
97
|
+
#
|
|
98
|
+
# @param message [String] The warning
|
|
99
|
+
# @return [void]
|
|
100
|
+
def self.warn_once(message)
|
|
101
|
+
warned = cache[:warned] ||= Set.new
|
|
102
|
+
Jekyll.logger.warn(Constants::LOG_PREFIX, message) if warned.add?(message)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Drops every cached result and logged warning. Registered on
|
|
106
|
+
# :site, :after_reset below, so each build (including each
|
|
107
|
+
# `jekyll serve --watch` rebuild) starts from the config as it is then.
|
|
108
|
+
#
|
|
109
|
+
# @return [void]
|
|
110
|
+
def self.reset!
|
|
111
|
+
@cache = nil
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Holds results strongly: a weak map would let GC drop them mid-build
|
|
115
|
+
# and repeat every warning. reset! clears it at the start of each build.
|
|
116
|
+
def self.cache
|
|
117
|
+
@cache ||= {}
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def self.check_defaults_values(values, source)
|
|
121
|
+
Schema.normalize(values[PAGE_DATA_KEY], source: source) if values.key?(PAGE_DATA_KEY)
|
|
122
|
+
Schema.boolean_flag(values[PAGE_FLAG_KEY], PAGE_FLAG_KEY, source) if values.key?(PAGE_FLAG_KEY)
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def self.resolve_page(page)
|
|
126
|
+
own = page.data[PAGE_DATA_KEY] if page.data.respond_to?(:key?) && page.data.key?(PAGE_DATA_KEY)
|
|
127
|
+
own_settings = Schema.normalize(own, source: source_of(page))
|
|
128
|
+
Jekyll::Utils.deep_merge_hashes(defaults_for(page), own_settings)
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# The page's merged `with_locales_data` defaults, cleaned without
|
|
132
|
+
# warnings (check_defaults has warned about each entry already), or {}
|
|
133
|
+
# when the page has no site to look them up in (as in some specs).
|
|
134
|
+
def self.defaults_for(page)
|
|
135
|
+
return {} unless page.is_a?(Jekyll::Page)
|
|
136
|
+
|
|
137
|
+
site = page.site
|
|
138
|
+
return {} unless site.respond_to?(:frontmatter_defaults)
|
|
139
|
+
|
|
140
|
+
check_defaults(site.config) if site.respond_to?(:config) && site.config.is_a?(Hash)
|
|
141
|
+
Schema.normalize(site.frontmatter_defaults.all(page.relative_path, :pages)[PAGE_DATA_KEY], source: nil)
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def self.source_of(page)
|
|
145
|
+
path = page.relative_path if page.respond_to?(:relative_path)
|
|
146
|
+
path = page.path if (path.nil? || path.empty?) && page.respond_to?(:path)
|
|
147
|
+
path.nil? || path.empty? ? 'page front matter' : path
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def self.deep_freeze(value)
|
|
151
|
+
case value
|
|
152
|
+
when Hash then value.each_value { |v| deep_freeze(v) }.freeze
|
|
153
|
+
when Array then value.each { |v| deep_freeze(v) }.freeze
|
|
154
|
+
else value
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
private_class_method :cache, :check_defaults_values, :resolve_page, :defaults_for, :source_of, :deep_freeze
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# Schema's constants read the key names, defined above.
|
|
163
|
+
require_relative 'configuration/schema'
|
|
164
|
+
|
|
165
|
+
Jekyll::Hooks.register(:site, :after_reset) { Jekyll::L10n::Configuration.reset! }
|
|
@@ -12,6 +12,10 @@ module Jekyll
|
|
|
12
12
|
#
|
|
13
13
|
# @see PageLocalesConfig for configuration-specific defaults
|
|
14
14
|
module Constants
|
|
15
|
+
# Prefix for every log line the plugin writes.
|
|
16
|
+
# @return [String]
|
|
17
|
+
LOG_PREFIX = 'Localization'
|
|
18
|
+
|
|
15
19
|
# ## Text Validation Constants
|
|
16
20
|
|
|
17
21
|
# Minimum length for translatable text strings (shorter strings not extracted)
|
|
@@ -100,31 +104,52 @@ module Jekyll
|
|
|
100
104
|
# @return [Integer] 60
|
|
101
105
|
LOG_TRUNCATE_LONG = 60
|
|
102
106
|
|
|
103
|
-
# ## Debug Logger Context Extraction
|
|
104
|
-
|
|
105
|
-
# Number of characters to show before/after difference in debug output
|
|
106
|
-
# @return [Integer] 10
|
|
107
|
-
BACKTRACE_CONTEXT_LENGTH = 10
|
|
108
|
-
|
|
109
107
|
# ## Default Configuration Values
|
|
110
108
|
|
|
111
109
|
# Default directory for storing PO translation files
|
|
112
110
|
# @return [String] "_locales"
|
|
113
111
|
DEFAULT_LOCALES_DIR = '_locales'
|
|
114
112
|
|
|
113
|
+
# Default for `extract_on_build`: extract strings on every build
|
|
114
|
+
# @return [Boolean] true
|
|
115
|
+
DEFAULT_EXTRACT_ON_BUILD = true
|
|
116
|
+
|
|
117
|
+
# Default for `update_compendium`: merge page PO files into compendia
|
|
118
|
+
# @return [Boolean] true
|
|
119
|
+
DEFAULT_UPDATE_COMPENDIUM = true
|
|
120
|
+
|
|
121
|
+
# Default for `logging.show_statistics`: print the extraction summary
|
|
122
|
+
# @return [Boolean] true
|
|
123
|
+
DEFAULT_SHOW_STATISTICS = true
|
|
124
|
+
|
|
125
|
+
# Default for `logging.debug`: no per-lookup translation log
|
|
126
|
+
# @return [Boolean] false
|
|
127
|
+
DEFAULT_DEBUG_LOGGING = false
|
|
128
|
+
|
|
129
|
+
# Default for `logging.trace`: no per-entry LibreTranslate log
|
|
130
|
+
# @return [Boolean] false
|
|
131
|
+
DEFAULT_TRACE_LOGGING = false
|
|
132
|
+
|
|
133
|
+
# Default for `jekyll_l10n.incremental`: rebuild every localized page
|
|
134
|
+
# @return [Boolean] false
|
|
135
|
+
DEFAULT_INCREMENTAL = false
|
|
136
|
+
|
|
115
137
|
# Default fallback mode when translation is not found
|
|
116
138
|
# @return [String] "english"
|
|
117
139
|
DEFAULT_FALLBACK_MODE = FALLBACK_MODE_ENGLISH
|
|
118
140
|
|
|
141
|
+
# Every accepted value of `translation.fallback`
|
|
142
|
+
# @return [Array<String>]
|
|
143
|
+
FALLBACK_MODES = [FALLBACK_MODE_ENGLISH, FALLBACK_MODE_MARKER, FALLBACK_MODE_EMPTY].freeze
|
|
144
|
+
|
|
119
145
|
# Default HTML attributes to extract from elements (can be overridden per-page)
|
|
120
146
|
# @return [Array<String>] ["title", "alt", "aria-label", "placeholder", "aria-description"]
|
|
121
147
|
DEFAULT_TRANSLATABLE_ATTRIBUTES = %w[title alt aria-label placeholder aria-description].freeze
|
|
122
148
|
|
|
123
|
-
#
|
|
124
|
-
#
|
|
125
|
-
# Prevents MT-corrupted class strings from reaching the rendered page.
|
|
149
|
+
# Default CSS selectors of elements left out of extraction (can be
|
|
150
|
+
# overridden per-page with `extraction.exclude_selectors`)
|
|
126
151
|
# @return [Array<String>]
|
|
127
|
-
|
|
152
|
+
DEFAULT_EXCLUDE_SELECTORS = ['script', 'style', 'code.language-plaintext', 'pre code'].freeze
|
|
128
153
|
|
|
129
154
|
# ## LibreTranslate Integration Defaults
|
|
130
155
|
|
|
@@ -142,6 +167,11 @@ module Jekyll
|
|
|
142
167
|
# @return [Integer] 50
|
|
143
168
|
DEFAULT_LIBRETRANSLATE_BATCH_SIZE = 50
|
|
144
169
|
|
|
170
|
+
# Default number of requests sent to LibreTranslate at once. Matches the
|
|
171
|
+
# server's default worker count, which translates one request per worker.
|
|
172
|
+
# @return [Integer] 4
|
|
173
|
+
DEFAULT_LIBRETRANSLATE_CONCURRENCY = 4
|
|
174
|
+
|
|
145
175
|
# Default number of retry attempts for failed LibreTranslate requests
|
|
146
176
|
# @return [Integer] 3
|
|
147
177
|
DEFAULT_LIBRETRANSLATE_RETRY_ATTEMPTS = 3
|
|
@@ -156,6 +186,11 @@ module Jekyll
|
|
|
156
186
|
# @return [Boolean] false
|
|
157
187
|
DEFAULT_LIBRETRANSLATE_STOP_ON_ERROR = false
|
|
158
188
|
|
|
189
|
+
# Whether machine translations are written as fuzzy entries, which
|
|
190
|
+
# pages don't use until a person reviews them and removes the flag.
|
|
191
|
+
# @return [Boolean]
|
|
192
|
+
DEFAULT_LIBRETRANSLATE_MARK_FUZZY = false
|
|
193
|
+
|
|
159
194
|
# Default interval for logging LibreTranslate translation progress
|
|
160
195
|
# Progress is logged every N entries. Set to 0 to disable.
|
|
161
196
|
# @return [Integer] 10 (log every 10 entries)
|
data/lib/jekyll-l10n/errors.rb
CHANGED
|
@@ -55,6 +55,13 @@ module Jekyll
|
|
|
55
55
|
# Common causes include network errors, API unavailability, invalid API keys,
|
|
56
56
|
# or rate limiting.
|
|
57
57
|
class LibreTranslateError < BaseError; end
|
|
58
|
+
|
|
59
|
+
# Error raised when a locale's translation coverage is below
|
|
60
|
+
# `translation.min_coverage`
|
|
61
|
+
#
|
|
62
|
+
# Fails the build so a CI job notices missing translations instead of
|
|
63
|
+
# publishing pages that fall back to the original text.
|
|
64
|
+
class CoverageError < BaseError; end
|
|
58
65
|
end
|
|
59
66
|
end
|
|
60
67
|
end
|