jekyll-l10n 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +44 -0
- data/lib/jekyll-l10n/configuration/schema.rb +308 -0
- data/lib/jekyll-l10n/configuration.rb +165 -0
- data/lib/jekyll-l10n/constants.rb +45 -10
- data/lib/jekyll-l10n/errors.rb +7 -0
- data/lib/jekyll-l10n/extraction/compendium_merger.rb +9 -11
- data/lib/jekyll-l10n/extraction/compendium_translator.rb +27 -11
- data/lib/jekyll-l10n/extraction/config_loader.rb +4 -4
- data/lib/jekyll-l10n/extraction/coverage_report.rb +129 -0
- data/lib/jekyll-l10n/extraction/dom_text_extractor.rb +2 -31
- data/lib/jekyll-l10n/extraction/extractor.rb +48 -28
- data/lib/jekyll-l10n/extraction/html_string_extractor.rb +24 -10
- data/lib/jekyll-l10n/extraction/logger.rb +6 -5
- data/lib/jekyll-l10n/extraction/result_saver.rb +5 -10
- data/lib/jekyll-l10n/instrumentation.rb +172 -139
- data/lib/jekyll-l10n/jekyll/generator.rb +10 -22
- data/lib/jekyll-l10n/jekyll/localized_page.rb +27 -49
- data/lib/jekyll-l10n/jekyll/post_write_html_reprocessor.rb +42 -25
- data/lib/jekyll-l10n/jekyll/post_write_processor.rb +2 -1
- data/lib/jekyll-l10n/jekyll/regeneration_checker.rb +16 -10
- data/lib/jekyll-l10n/jekyll/url_filter.rb +8 -7
- data/lib/jekyll-l10n/po_file/entry_parser.rb +155 -0
- data/lib/jekyll-l10n/po_file/fuzzy_matcher.rb +67 -53
- data/lib/jekyll-l10n/po_file/loader.rb +20 -11
- data/lib/jekyll-l10n/po_file/manager.rb +32 -17
- data/lib/jekyll-l10n/po_file/merger.rb +2 -1
- data/lib/jekyll-l10n/po_file/path_builder.rb +11 -1
- data/lib/jekyll-l10n/po_file/reader.rb +40 -508
- data/lib/jekyll-l10n/po_file/writer.rb +31 -16
- data/lib/jekyll-l10n/translation/block_text_extractor.rb +3 -7
- data/lib/jekyll-l10n/translation/html_translator.rb +136 -157
- data/lib/jekyll-l10n/translation/libre_translate_languages.rb +73 -0
- data/lib/jekyll-l10n/translation/libre_translator.rb +134 -78
- data/lib/jekyll-l10n/translation/request_slots.rb +69 -0
- data/lib/jekyll-l10n/translation/translation_progress.rb +77 -0
- data/lib/jekyll-l10n/translation/translator.rb +20 -17
- data/lib/jekyll-l10n/utils/baseurl_normalizer.rb +108 -0
- data/lib/jekyll-l10n/utils/debug_logger.rb +4 -51
- data/lib/jekyll-l10n/utils/error_handler.rb +9 -24
- data/lib/jekyll-l10n/utils/external_link_icon_preserver.rb +5 -62
- data/lib/jekyll-l10n/utils/html_parser.rb +24 -31
- data/lib/jekyll-l10n/utils/html_text_utils.rb +146 -91
- data/lib/jekyll-l10n/utils/logger_formatter.rb +6 -4
- data/lib/jekyll-l10n/utils/msgstr_validator.rb +74 -0
- data/lib/jekyll-l10n/utils/page_locales_config.rb +126 -107
- data/lib/jekyll-l10n/utils/po_entry_converter.rb +0 -43
- data/lib/jekyll-l10n/utils/site_config_accessor.rb +56 -54
- data/lib/jekyll-l10n/utils/translation_resolver.rb +56 -6
- data/lib/jekyll-l10n/utils/url_path_builder.rb +5 -27
- data/lib/jekyll-l10n/utils/url_transformer.rb +5 -11
- data/lib/jekyll-l10n/version.rb +1 -1
- data/lib/jekyll-l10n.rb +9 -6
- metadata +12 -3
|
@@ -4,7 +4,6 @@ require 'fileutils'
|
|
|
4
4
|
require_relative '../po_file/manager'
|
|
5
5
|
require_relative '../po_file/fuzzy_matcher'
|
|
6
6
|
require_relative '../utils/page_locales_config'
|
|
7
|
-
require_relative '../utils/site_config_accessor'
|
|
8
7
|
|
|
9
8
|
module Jekyll
|
|
10
9
|
module L10n
|
|
@@ -35,8 +34,6 @@ module Jekyll
|
|
|
35
34
|
# @param site [Jekyll::Site] Jekyll site object
|
|
36
35
|
def initialize(site)
|
|
37
36
|
@site = site
|
|
38
|
-
with_locales_data = SiteConfigAccessor.extract_locales_data(@site)
|
|
39
|
-
@site_config = PageLocalesConfig.new({ 'with_locales_data' => with_locales_data })
|
|
40
37
|
end
|
|
41
38
|
|
|
42
39
|
# Merge page-specific PO files into compendia for all locales.
|
|
@@ -49,8 +46,11 @@ module Jekyll
|
|
|
49
46
|
# @param config [PageLocalesConfig] Localization configuration with locales list
|
|
50
47
|
# @return [void]
|
|
51
48
|
def merge_compendia(po_manager, config)
|
|
49
|
+
# Fuzzy scores depend only on the two English strings, and every
|
|
50
|
+
# locale compares the same new strings, so they are scored once per run.
|
|
51
|
+
@fuzzy_scores = {}
|
|
52
52
|
config.locales.each do |locale|
|
|
53
|
-
process_locale(locale, po_manager
|
|
53
|
+
process_locale(locale, po_manager)
|
|
54
54
|
end
|
|
55
55
|
end
|
|
56
56
|
|
|
@@ -58,9 +58,8 @@ module Jekyll
|
|
|
58
58
|
|
|
59
59
|
# Process a single locale: merge existing compendium with new page translations
|
|
60
60
|
# Skips writing when translations are unchanged to avoid unnecessary file rewrites
|
|
61
|
-
def process_locale(locale, po_manager
|
|
62
|
-
|
|
63
|
-
existing_compendium = load_existing_compendium(compendium_path)
|
|
61
|
+
def process_locale(locale, po_manager)
|
|
62
|
+
existing_compendium = load_existing_compendium(po_manager.compendium_path(locale))
|
|
64
63
|
merged = po_manager.merge_po_files(locale)
|
|
65
64
|
|
|
66
65
|
combined = build_combined_hash(existing_compendium)
|
|
@@ -70,7 +69,7 @@ module Jekyll
|
|
|
70
69
|
combined_entries = format_compendium_entries(combined)
|
|
71
70
|
po_manager.save_compendium(locale, combined_entries)
|
|
72
71
|
end
|
|
73
|
-
cleanup_locale_directory(locale
|
|
72
|
+
cleanup_locale_directory(po_manager.locale_dir(locale))
|
|
74
73
|
end
|
|
75
74
|
|
|
76
75
|
# Check if compendium content is unchanged (same msgids and msgstrs)
|
|
@@ -146,7 +145,7 @@ module Jekyll
|
|
|
146
145
|
end
|
|
147
146
|
|
|
148
147
|
def resolve_new_entry(combined, msgid, entry, fuzzy_candidates, matched_old_msgids)
|
|
149
|
-
match = PoFuzzyMatcher.find_match(msgid, fuzzy_candidates)
|
|
148
|
+
match = PoFuzzyMatcher.find_match(msgid, fuzzy_candidates, scores: @fuzzy_scores)
|
|
150
149
|
if match
|
|
151
150
|
combined[msgid] = create_fuzzy_entry(entry, match)
|
|
152
151
|
matched_old_msgids << match[:msgid]
|
|
@@ -195,8 +194,7 @@ module Jekyll
|
|
|
195
194
|
|
|
196
195
|
# IMPORTANT: Remove locale-specific directory after compendium merge
|
|
197
196
|
# This cleans up page-specific PO files that have been merged into the compendium
|
|
198
|
-
def cleanup_locale_directory(
|
|
199
|
-
locale_dir = File.join(@site.source, config.locales_dir, locale)
|
|
197
|
+
def cleanup_locale_directory(locale_dir)
|
|
200
198
|
FileUtils.rm_rf(locale_dir) if File.directory?(locale_dir)
|
|
201
199
|
end
|
|
202
200
|
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative '../constants'
|
|
4
|
+
require_relative '../instrumentation'
|
|
3
5
|
require_relative '../po_file/manager'
|
|
4
6
|
require_relative '../utils/page_locales_config'
|
|
5
7
|
require_relative '../utils/po_entry_converter'
|
|
6
|
-
require_relative '../utils/site_config_accessor'
|
|
7
8
|
require_relative '../utils/logger_formatter'
|
|
8
9
|
require_relative '../translation/libre_translator'
|
|
9
10
|
|
|
@@ -33,18 +34,19 @@ module Jekyll
|
|
|
33
34
|
# @param site [Jekyll::Site] Jekyll site object
|
|
34
35
|
def initialize(site)
|
|
35
36
|
@site = site
|
|
36
|
-
with_locales_data = SiteConfigAccessor.extract_locales_data(@site)
|
|
37
|
-
@site_config = PageLocalesConfig.new({ 'with_locales_data' => with_locales_data })
|
|
38
37
|
end
|
|
39
38
|
|
|
40
39
|
# Translate compendia for all configured locales.
|
|
41
40
|
#
|
|
42
41
|
# Checks if LibreTranslate is enabled. If so, for each configured locale,
|
|
43
42
|
# loads the compendium PO file, identifies untranslated entries, sends them
|
|
44
|
-
# to LibreTranslate API, and saves the updated file.
|
|
43
|
+
# to LibreTranslate API, and saves the updated file. Locales are
|
|
44
|
+
# translated at once; their requests share libretranslate_concurrency.
|
|
45
45
|
#
|
|
46
46
|
# @param config [PageLocalesConfig] Localization configuration
|
|
47
47
|
# @return [void]
|
|
48
|
+
# @raise [LibreTranslator::TranslationError] If a locale failed, once
|
|
49
|
+
# every locale has finished; the locales that finished are saved
|
|
48
50
|
def translate_compendia(config)
|
|
49
51
|
po_manager = PoFileManager.new(@site, config.locales_dir)
|
|
50
52
|
translate_compendia_for_locale(po_manager, config)
|
|
@@ -59,15 +61,29 @@ module Jekyll
|
|
|
59
61
|
log_translation_start(config)
|
|
60
62
|
translator = LibreTranslator.new(config)
|
|
61
63
|
|
|
62
|
-
config.locales.
|
|
63
|
-
|
|
64
|
+
threads = config.locales.map do |locale|
|
|
65
|
+
Instrumentation.traced_thread do
|
|
66
|
+
Thread.current.report_on_exception = false
|
|
67
|
+
process_single_locale(locale, translator, po_manager)
|
|
68
|
+
end
|
|
64
69
|
end
|
|
70
|
+
join_locales(threads)
|
|
65
71
|
|
|
66
72
|
log_translation_complete(config)
|
|
67
73
|
end
|
|
68
74
|
|
|
69
|
-
def
|
|
70
|
-
|
|
75
|
+
def join_locales(threads)
|
|
76
|
+
first_error = nil
|
|
77
|
+
threads.each do |thread|
|
|
78
|
+
thread.join
|
|
79
|
+
rescue StandardError => e
|
|
80
|
+
first_error ||= e
|
|
81
|
+
end
|
|
82
|
+
raise first_error if first_error
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def process_single_locale(locale, translator, po_manager)
|
|
86
|
+
compendium_path = po_manager.compendium_path(locale)
|
|
71
87
|
LoggerFormatter.debug_if_enabled('CompendiumTranslator',
|
|
72
88
|
"Processing compendium file: #{compendium_path}")
|
|
73
89
|
return unless File.exist?(compendium_path)
|
|
@@ -119,17 +135,17 @@ module Jekyll
|
|
|
119
135
|
|
|
120
136
|
def log_translation_start(config)
|
|
121
137
|
locales = config.locales.join(', ')
|
|
122
|
-
Jekyll.logger.info
|
|
138
|
+
Jekyll.logger.info Constants::LOG_PREFIX,
|
|
123
139
|
"Starting LibreTranslate translation for locales: #{locales}"
|
|
124
140
|
end
|
|
125
141
|
|
|
126
142
|
def log_translation_complete(config)
|
|
127
143
|
locales = config.locales.join(', ')
|
|
128
|
-
Jekyll.logger.info
|
|
144
|
+
Jekyll.logger.info Constants::LOG_PREFIX,
|
|
129
145
|
"LibreTranslate translation complete for locales: #{locales}"
|
|
130
146
|
end
|
|
131
147
|
|
|
132
|
-
private :translate_compendia_for_locale, :process_single_locale,
|
|
148
|
+
private :translate_compendia_for_locale, :join_locales, :process_single_locale,
|
|
133
149
|
:log_compendia_enabled_check, :log_compendium_stats,
|
|
134
150
|
:log_translation_complete_for_locale, :log_compendium_saved,
|
|
135
151
|
:count_empty_entries, :log_translation_start, :log_translation_complete
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative '../configuration'
|
|
3
4
|
require_relative '../utils/page_locales_config'
|
|
4
5
|
require_relative '../utils/locale_url_formatter'
|
|
5
6
|
require_relative '../utils/site_config_accessor'
|
|
@@ -51,7 +52,7 @@ module Jekyll
|
|
|
51
52
|
# @return [Hash, nil] The page data, or nil when no localized page matches
|
|
52
53
|
def find_page_config_for_file(file_path)
|
|
53
54
|
@site.pages.each do |page|
|
|
54
|
-
next unless page
|
|
55
|
+
next unless Configuration.localizable?(page)
|
|
55
56
|
|
|
56
57
|
page_output = page.output_ext ? page.destination('') : page.destination('/')
|
|
57
58
|
next unless file_path.end_with?(page_output.sub(%r{/$}, '/index.html'))
|
|
@@ -77,10 +78,9 @@ module Jekyll
|
|
|
77
78
|
|
|
78
79
|
# @param config [PageLocalesConfig] The page's settings
|
|
79
80
|
# @return [Array<String>] CSS selectors of elements to leave out of
|
|
80
|
-
# extraction
|
|
81
|
+
# extraction (see PageLocalesConfig#exclude_selectors)
|
|
81
82
|
def extract_exclude_selectors(config)
|
|
82
|
-
config.
|
|
83
|
-
['script', 'style', 'code.language-plaintext', 'pre code']
|
|
83
|
+
config.exclude_selectors
|
|
84
84
|
end
|
|
85
85
|
end
|
|
86
86
|
end
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative '../constants'
|
|
4
|
+
require_relative '../errors'
|
|
5
|
+
require_relative '../po_file/path_builder'
|
|
6
|
+
require_relative '../po_file/reader'
|
|
7
|
+
require_relative '../utils/msgstr_validator'
|
|
8
|
+
|
|
9
|
+
module Jekyll
|
|
10
|
+
module L10n
|
|
11
|
+
# Counts how much of each locale's compendium is translated.
|
|
12
|
+
#
|
|
13
|
+
# Each entry counts once, as translated, fuzzy, untranslated, or invalid
|
|
14
|
+
# (a msgstr whose markup MsgstrValidator rejects, so pages use the
|
|
15
|
+
# fallback). Extractor warns about each invalid entry, prints the counts
|
|
16
|
+
# with the extraction summary, and fails the build when a locale is below
|
|
17
|
+
# `translation.min_coverage`.
|
|
18
|
+
#
|
|
19
|
+
# Only entries whose msgid this build extracted count: a compendium also
|
|
20
|
+
# keeps entries for text the site no longer has. The counts come from the
|
|
21
|
+
# compendium files as they are after extraction, so with
|
|
22
|
+
# `update_compendium: false` they leave out strings no compendium holds
|
|
23
|
+
# yet.
|
|
24
|
+
#
|
|
25
|
+
# @example
|
|
26
|
+
# report = CoverageReport.new(site, SiteConfigAccessor.site_config(site, all_locales: true),
|
|
27
|
+
# extracted_msgids)
|
|
28
|
+
# report.warn_invalid
|
|
29
|
+
# report.log
|
|
30
|
+
# report.enforce!(90) # raises Errors::CoverageError below 90%
|
|
31
|
+
class CoverageReport
|
|
32
|
+
# Entry counts for one locale.
|
|
33
|
+
Counts = Struct.new(:locale, :translated, :fuzzy, :untranslated, :invalid) do
|
|
34
|
+
# @return [Integer] Entries in the compendium
|
|
35
|
+
def total
|
|
36
|
+
translated + fuzzy + untranslated + invalid
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# @return [Float] Translated entries as a percentage; 100 when the
|
|
40
|
+
# compendium is empty
|
|
41
|
+
def percent
|
|
42
|
+
total.zero? ? 100.0 : translated * 100.0 / total
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# @return [String] e.g. "fr: 92.5% translated (1200 of 1297; 40 fuzzy, 50 untranslated, 7 invalid)"
|
|
46
|
+
def to_s
|
|
47
|
+
"#{locale}: #{percent.round(1)}% translated (#{translated} of #{total}; " \
|
|
48
|
+
"#{fuzzy} fuzzy, #{untranslated} untranslated, #{invalid} invalid)"
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# @param site [Jekyll::Site]
|
|
53
|
+
# @param config [PageLocalesConfig] The site config, for locales and locales_dir
|
|
54
|
+
# @param msgids [Set<String>] The msgids this build extracted
|
|
55
|
+
def initialize(site, config, msgids)
|
|
56
|
+
@site = site
|
|
57
|
+
@config = config
|
|
58
|
+
@msgids = msgids
|
|
59
|
+
@invalid = Hash.new { |hash, locale| hash[locale] = [] }
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# @return [Array<Counts>] One entry per locale that has a compendium
|
|
63
|
+
def counts
|
|
64
|
+
@counts ||= @config.locales.filter_map do |locale|
|
|
65
|
+
path = PoPathBuilder.build(@site.source, @config.locales_dir, locale, nil)
|
|
66
|
+
count(locale, PoFileReader.parse_for_merge(path)) if File.exist?(path)
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Warn about each entry whose msgstr markup does not match its msgid.
|
|
71
|
+
#
|
|
72
|
+
# @return [void]
|
|
73
|
+
def warn_invalid
|
|
74
|
+
counts.each do |locale_counts|
|
|
75
|
+
@invalid[locale_counts.locale].each do |msgid|
|
|
76
|
+
Jekyll.logger.warn Constants::LOG_PREFIX,
|
|
77
|
+
"#{locale_counts.locale}: msgstr markup does not match its msgid, " \
|
|
78
|
+
"so pages show the fallback: #{msgid[0..Constants::LOG_TRUNCATE_LONG]}"
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Print one line per locale.
|
|
84
|
+
#
|
|
85
|
+
# @return [void]
|
|
86
|
+
def log
|
|
87
|
+
return if counts.empty?
|
|
88
|
+
|
|
89
|
+
Jekyll.logger.info Constants::LOG_PREFIX, 'Translation coverage:'
|
|
90
|
+
counts.each { |locale_counts| Jekyll.logger.info Constants::LOG_PREFIX, " #{locale_counts}" }
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# @param minimum [Integer, nil] Required percentage, or nil for none
|
|
94
|
+
# @return [void]
|
|
95
|
+
# @raise [Errors::CoverageError] When any locale is below the minimum
|
|
96
|
+
def enforce!(minimum)
|
|
97
|
+
return if minimum.nil?
|
|
98
|
+
|
|
99
|
+
below = counts.select { |locale_counts| locale_counts.percent < minimum }
|
|
100
|
+
return if below.empty?
|
|
101
|
+
|
|
102
|
+
raise Errors::CoverageError,
|
|
103
|
+
"Translation coverage is below translation.min_coverage (#{minimum}%): #{below.join('; ')}"
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
private
|
|
107
|
+
|
|
108
|
+
def count(locale, entries)
|
|
109
|
+
tally = Hash.new(0)
|
|
110
|
+
entries.each do |msgid, entry|
|
|
111
|
+
next unless @msgids.include?(msgid)
|
|
112
|
+
|
|
113
|
+
entry_state = state(msgid, entry)
|
|
114
|
+
tally[entry_state] += 1
|
|
115
|
+
@invalid[locale] << msgid if entry_state == :invalid
|
|
116
|
+
end
|
|
117
|
+
Counts.new(locale, tally[:translated], tally[:fuzzy], tally[:untranslated], tally[:invalid])
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def state(msgid, entry)
|
|
121
|
+
if entry[:fuzzy] then :fuzzy
|
|
122
|
+
elsif entry[:msgstr].to_s.empty? then :untranslated
|
|
123
|
+
elsif MsgstrValidator.valid?(msgid, entry[:msgstr]) then :translated
|
|
124
|
+
else :invalid
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
end
|
|
@@ -3,8 +3,6 @@
|
|
|
3
3
|
require_relative '../utils/text_normalizer'
|
|
4
4
|
require_relative '../utils/xpath_reference_generator'
|
|
5
5
|
require_relative '../utils/html_text_utils'
|
|
6
|
-
require_relative '../utils/html_elements'
|
|
7
|
-
require_relative '../utils/text_validator'
|
|
8
6
|
|
|
9
7
|
module Jekyll
|
|
10
8
|
module L10n
|
|
@@ -27,7 +25,7 @@ module Jekyll
|
|
|
27
25
|
# entry = DomTextExtractor.extract(node, 'docs/index.html', '_site')
|
|
28
26
|
# # Returns hash with :msgid, :msgstr, :reference if valid text found
|
|
29
27
|
module DomTextExtractor
|
|
30
|
-
|
|
28
|
+
module_function
|
|
31
29
|
|
|
32
30
|
# Extract text content from an HTML element.
|
|
33
31
|
#
|
|
@@ -56,34 +54,7 @@ module Jekyll
|
|
|
56
54
|
end
|
|
57
55
|
|
|
58
56
|
def extract_block_text(node)
|
|
59
|
-
|
|
60
|
-
return nil if HtmlTextUtils.layout_block_children?(node)
|
|
61
|
-
|
|
62
|
-
text = HtmlTextUtils.extract_with_inline_tags(node)
|
|
63
|
-
TextValidator.valid?(text) ? text : nil
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
def only_contains_block_elements?(node)
|
|
67
|
-
node.children.each do |child|
|
|
68
|
-
return false if non_empty_text?(child)
|
|
69
|
-
return false if non_block_element?(child)
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
block_element_children?(node)
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
private
|
|
76
|
-
|
|
77
|
-
def non_empty_text?(child)
|
|
78
|
-
child.text? && !child.content.strip.empty?
|
|
79
|
-
end
|
|
80
|
-
|
|
81
|
-
def non_block_element?(child)
|
|
82
|
-
child.element? && !HtmlTextUtils::ALL_BLOCK_ELEMENTS.include?(child.name)
|
|
83
|
-
end
|
|
84
|
-
|
|
85
|
-
def block_element_children?(node)
|
|
86
|
-
node.children.any? { |c| c.element? && HtmlTextUtils::ALL_BLOCK_ELEMENTS.include?(c.name) }
|
|
57
|
+
HtmlTextUtils.block_text(node)
|
|
87
58
|
end
|
|
88
59
|
end
|
|
89
60
|
end
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'set'
|
|
4
|
+
require_relative '../configuration'
|
|
5
|
+
require_relative '../constants'
|
|
3
6
|
require_relative 'config_loader'
|
|
4
7
|
require_relative 'result_saver'
|
|
5
8
|
require_relative 'logger'
|
|
6
9
|
require_relative 'html_string_extractor'
|
|
10
|
+
require_relative 'coverage_report'
|
|
7
11
|
require_relative '../utils/file_operations'
|
|
8
12
|
require_relative '../utils/site_config_accessor'
|
|
9
13
|
require_relative '../utils/error_handler'
|
|
@@ -20,7 +24,10 @@ module Jekyll
|
|
|
20
24
|
# 3. Merges those files into one compendium per locale (unless
|
|
21
25
|
# `update_compendium` is false). The merge deletes the page files.
|
|
22
26
|
# 4. Machine-translates the compendia when LibreTranslate is on.
|
|
23
|
-
# 5. Prints a summary
|
|
27
|
+
# 5. Prints a summary and each locale's translation coverage unless
|
|
28
|
+
# `logging.show_statistics` is false.
|
|
29
|
+
# 6. Fails the build when a locale's coverage is below
|
|
30
|
+
# `translation.min_coverage`.
|
|
24
31
|
#
|
|
25
32
|
# It always processes the whole site. Incremental mode does not affect it.
|
|
26
33
|
#
|
|
@@ -43,19 +50,22 @@ module Jekyll
|
|
|
43
50
|
@dest = SiteConfigAccessor.dest(@site)
|
|
44
51
|
@config_loader = ExtractionConfigLoader.new(@site, @dest)
|
|
45
52
|
@result_saver = ExtractionResultSaver.new(@site)
|
|
53
|
+
@extracted_msgids = Set.new
|
|
46
54
|
end
|
|
47
55
|
|
|
48
56
|
# Runs all the steps in the class description.
|
|
49
57
|
#
|
|
50
58
|
# @return [Hash<Symbol, Integer>] Counts under :files_processed,
|
|
51
59
|
# :strings_extracted, and :po_files_created (files created or updated)
|
|
60
|
+
# @raise [Errors::CoverageError] When a locale is below `translation.min_coverage`
|
|
52
61
|
def extract_site
|
|
53
|
-
Jekyll.logger.info
|
|
62
|
+
Jekyll.logger.info Constants::LOG_PREFIX, 'Extracting translatable strings...'
|
|
54
63
|
start_time = Time.now
|
|
55
64
|
stats = process_all_html_files
|
|
56
65
|
@result_saver.finalize_compendia
|
|
57
66
|
translate_all_compendia
|
|
58
67
|
ExtractionLogger.log_summary(stats, Time.now - start_time) if show_statistics?
|
|
68
|
+
check_coverage
|
|
59
69
|
stats
|
|
60
70
|
end
|
|
61
71
|
|
|
@@ -84,20 +94,6 @@ module Jekyll
|
|
|
84
94
|
end
|
|
85
95
|
end
|
|
86
96
|
|
|
87
|
-
def process_file(file_path)
|
|
88
|
-
return default_stats unless @config_loader.valid_for_extraction?(file_path)
|
|
89
|
-
|
|
90
|
-
config = @config_loader.load_page_config(file_path)
|
|
91
|
-
entries = extract_strings_from_file(file_path, config)
|
|
92
|
-
return default_stats if entries.empty?
|
|
93
|
-
|
|
94
|
-
page_path = construct_page_path(file_path)
|
|
95
|
-
@result_saver.save_results(config, entries, page_path)
|
|
96
|
-
rescue StandardError => e
|
|
97
|
-
ExtractionLogger.log_error(file_path, e)
|
|
98
|
-
default_stats
|
|
99
|
-
end
|
|
100
|
-
|
|
101
97
|
def default_stats
|
|
102
98
|
{ files_processed: 0, strings_extracted: 0, po_files_created: 0 }
|
|
103
99
|
end
|
|
@@ -108,32 +104,56 @@ module Jekyll
|
|
|
108
104
|
html = FileOperations.read_utf8(file_path)
|
|
109
105
|
exclude_selectors = @config_loader.extract_exclude_selectors(config)
|
|
110
106
|
|
|
111
|
-
extractor = HtmlStringExtractor.new(config.translatable_attributes, exclude_selectors
|
|
107
|
+
extractor = HtmlStringExtractor.new(config.translatable_attributes, exclude_selectors,
|
|
108
|
+
baseurl: @site.config['baseurl'] || '')
|
|
112
109
|
extractor.extract(html, @dest, file_path)
|
|
113
110
|
end
|
|
114
111
|
|
|
115
|
-
# Machine translation
|
|
116
|
-
#
|
|
112
|
+
# Machine translation is a site-wide step: it reads the `jekyll_l10n`
|
|
113
|
+
# settings and covers every locale the site uses.
|
|
114
|
+
#
|
|
115
|
+
# @return [PageLocalesConfig, nil] The site config, or nil when
|
|
116
|
+
# LibreTranslate is off
|
|
117
117
|
def find_libretranslate_config
|
|
118
118
|
return nil unless @site.respond_to?(:pages)
|
|
119
119
|
|
|
120
|
-
@site
|
|
121
|
-
|
|
120
|
+
config = SiteConfigAccessor.site_config(@site, all_locales: true)
|
|
121
|
+
config if config.libretranslate_enabled?
|
|
122
|
+
end
|
|
122
123
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
124
|
+
private
|
|
125
|
+
|
|
126
|
+
# per-page body called from the html_files loop in process_all_html_files
|
|
127
|
+
def process_file(file_path)
|
|
128
|
+
return default_stats unless @config_loader.valid_for_extraction?(file_path)
|
|
129
|
+
|
|
130
|
+
config = @config_loader.load_page_config(file_path)
|
|
131
|
+
entries = extract_strings_from_file(file_path, config)
|
|
132
|
+
return default_stats if entries.empty?
|
|
126
133
|
|
|
127
|
-
|
|
134
|
+
@extracted_msgids.merge(entries.map { |entry| entry[:msgid] })
|
|
135
|
+
|
|
136
|
+
page_path = construct_page_path(file_path)
|
|
137
|
+
@result_saver.save_results(config, entries, page_path)
|
|
138
|
+
rescue StandardError => e
|
|
139
|
+
ExtractionLogger.log_error(file_path, e)
|
|
140
|
+
default_stats
|
|
128
141
|
end
|
|
129
142
|
|
|
130
|
-
|
|
143
|
+
# Runs after machine translation, so its entries count, and outside
|
|
144
|
+
# ErrorHandler, so a CoverageError fails the build.
|
|
145
|
+
def check_coverage
|
|
146
|
+
config = SiteConfigAccessor.site_config(@site, all_locales: true)
|
|
147
|
+
report = CoverageReport.new(@site, config, @extracted_msgids)
|
|
148
|
+
report.warn_invalid
|
|
149
|
+
report.log if config.show_statistics?
|
|
150
|
+
report.enforce!(config.min_coverage)
|
|
151
|
+
end
|
|
131
152
|
|
|
132
153
|
# The summary covers the whole site, so read the site-wide setting, not a
|
|
133
154
|
# page's.
|
|
134
155
|
def show_statistics?
|
|
135
|
-
|
|
136
|
-
PageLocalesConfig.new('with_locales_data' => site_data).show_statistics?
|
|
156
|
+
SiteConfigAccessor.site_config(@site).show_statistics?
|
|
137
157
|
end
|
|
138
158
|
|
|
139
159
|
def construct_page_path(file_path)
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative '../constants'
|
|
3
4
|
require_relative 'dom_text_extractor'
|
|
4
5
|
require_relative 'dom_attribute_extractor'
|
|
5
6
|
require_relative 'logger'
|
|
7
|
+
require_relative '../utils/baseurl_normalizer'
|
|
8
|
+
require_relative '../utils/html_elements'
|
|
9
|
+
require_relative '../utils/html_parser'
|
|
6
10
|
|
|
7
11
|
module Jekyll
|
|
8
12
|
module L10n
|
|
@@ -27,7 +31,7 @@ module Jekyll
|
|
|
27
31
|
# entries = extractor.extract(html_content, '_site', 'docs/index.html')
|
|
28
32
|
# # Returns array of hash entries with :msgid, :msgstr, :reference keys
|
|
29
33
|
class HtmlStringExtractor
|
|
30
|
-
attr_reader :translatable_attrs, :exclude_selectors
|
|
34
|
+
attr_reader :translatable_attrs, :exclude_selectors, :baseurl
|
|
31
35
|
|
|
32
36
|
# Initialize a new HtmlStringExtractor.
|
|
33
37
|
#
|
|
@@ -35,9 +39,12 @@ module Jekyll
|
|
|
35
39
|
# (e.g., ['title', 'alt', 'aria-label', 'placeholder', 'aria-description'])
|
|
36
40
|
# @param exclude_selectors [Array<String>] CSS selectors for elements to skip
|
|
37
41
|
# during extraction (e.g., ['script', 'style', '.no-translate'])
|
|
38
|
-
|
|
42
|
+
# @param baseurl [String] (keyword) Site baseurl, kept out of msgids
|
|
43
|
+
# (see {BaseurlNormalizer})
|
|
44
|
+
def initialize(translatable_attrs, exclude_selectors, baseurl: '')
|
|
39
45
|
@translatable_attrs = translatable_attrs
|
|
40
46
|
@exclude_selectors = exclude_selectors
|
|
47
|
+
@baseurl = baseurl
|
|
41
48
|
end
|
|
42
49
|
|
|
43
50
|
# Extract translatable strings from HTML.
|
|
@@ -57,7 +64,8 @@ module Jekyll
|
|
|
57
64
|
def extract(html, dest, file_path)
|
|
58
65
|
entries = []
|
|
59
66
|
|
|
60
|
-
doc =
|
|
67
|
+
doc = HtmlParser.parse_document(html)
|
|
68
|
+
BaseurlNormalizer.strip(doc, @baseurl)
|
|
61
69
|
walk_dom(doc.root, file_path, entries, dest)
|
|
62
70
|
|
|
63
71
|
entries.uniq { |e| e[:msgid] }
|
|
@@ -65,17 +73,23 @@ module Jekyll
|
|
|
65
73
|
|
|
66
74
|
private
|
|
67
75
|
|
|
68
|
-
|
|
76
|
+
# Below a content element that has a msgid, inline text is translated only
|
|
77
|
+
# as part of that msgid, so it gets none of its own (covered). A nested
|
|
78
|
+
# content element is looked up by its own msgid, so it is extracted again
|
|
79
|
+
# and decides coverage for its own descendants.
|
|
80
|
+
def walk_dom(node, file_path, entries, dest, covered: false)
|
|
69
81
|
return if node.nil? || should_exclude_element?(node)
|
|
70
82
|
|
|
71
|
-
|
|
83
|
+
block = node.element? && HtmlElements::CONTENT_ELEMENTS.include?(node.name)
|
|
84
|
+
entry = process_text_content(node, file_path, entries, dest) if block || !covered
|
|
72
85
|
process_attributes(node, file_path, entries, dest)
|
|
73
|
-
process_children(node, file_path, entries, dest)
|
|
86
|
+
process_children(node, file_path, entries, dest, covered: block ? !entry.nil? : covered)
|
|
74
87
|
end
|
|
75
88
|
|
|
76
89
|
def process_text_content(node, file_path, entries, dest)
|
|
77
90
|
entry = DomTextExtractor.extract(node, file_path, dest)
|
|
78
91
|
entries << entry if entry
|
|
92
|
+
entry
|
|
79
93
|
end
|
|
80
94
|
|
|
81
95
|
def process_attributes(node, file_path, entries, dest)
|
|
@@ -83,8 +97,8 @@ module Jekyll
|
|
|
83
97
|
entries.concat(attr_entries)
|
|
84
98
|
end
|
|
85
99
|
|
|
86
|
-
def process_children(node, file_path, entries, dest)
|
|
87
|
-
node.children.each { |child| walk_dom(child, file_path, entries, dest) }
|
|
100
|
+
def process_children(node, file_path, entries, dest, covered: false)
|
|
101
|
+
node.children.each { |child| walk_dom(child, file_path, entries, dest, covered: covered) }
|
|
88
102
|
end
|
|
89
103
|
|
|
90
104
|
def should_exclude_element?(node)
|
|
@@ -92,10 +106,10 @@ module Jekyll
|
|
|
92
106
|
|
|
93
107
|
@exclude_selectors.any? { |selector| node.matches?(selector) }
|
|
94
108
|
rescue Nokogiri::CSS::SyntaxError => e
|
|
95
|
-
Jekyll.logger.warn
|
|
109
|
+
Jekyll.logger.warn Constants::LOG_PREFIX, "CSS selector syntax error: #{e.message}"
|
|
96
110
|
false
|
|
97
111
|
rescue StandardError => e
|
|
98
|
-
Jekyll.logger.warn
|
|
112
|
+
Jekyll.logger.warn Constants::LOG_PREFIX, "Selector matching error (continuing): #{e.message}"
|
|
99
113
|
false
|
|
100
114
|
end
|
|
101
115
|
end
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative '../constants'
|
|
3
4
|
require_relative '../utils/error_handler'
|
|
4
5
|
|
|
5
6
|
module Jekyll
|
|
@@ -39,11 +40,11 @@ module Jekyll
|
|
|
39
40
|
# @param elapsed [Float] Time elapsed in seconds
|
|
40
41
|
# @return [void]
|
|
41
42
|
def self.log_summary(stats, elapsed)
|
|
42
|
-
Jekyll.logger.info
|
|
43
|
-
Jekyll.logger.info
|
|
44
|
-
Jekyll.logger.info
|
|
45
|
-
Jekyll.logger.info
|
|
46
|
-
Jekyll.logger.info
|
|
43
|
+
Jekyll.logger.info Constants::LOG_PREFIX, 'Extraction complete:'
|
|
44
|
+
Jekyll.logger.info Constants::LOG_PREFIX, " Files processed: #{stats[:files_processed]}"
|
|
45
|
+
Jekyll.logger.info Constants::LOG_PREFIX, " Strings extracted: #{stats[:strings_extracted]}"
|
|
46
|
+
Jekyll.logger.info Constants::LOG_PREFIX, " PO files created/updated: #{stats[:po_files_created]}"
|
|
47
|
+
Jekyll.logger.info Constants::LOG_PREFIX, " Time: #{elapsed.round(2)}s"
|
|
47
48
|
end
|
|
48
49
|
end
|
|
49
50
|
end
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative '../configuration'
|
|
3
4
|
require_relative '../po_file/manager'
|
|
4
5
|
require_relative '../utils/page_locales_config'
|
|
5
6
|
require_relative '../utils/site_config_accessor'
|
|
@@ -33,8 +34,7 @@ module Jekyll
|
|
|
33
34
|
# @param site [Jekyll::Site] Jekyll site object
|
|
34
35
|
def initialize(site)
|
|
35
36
|
@site = site
|
|
36
|
-
|
|
37
|
-
@site_config = PageLocalesConfig.new({ 'with_locales_data' => with_locales_data })
|
|
37
|
+
@site_config = SiteConfigAccessor.site_config(@site)
|
|
38
38
|
end
|
|
39
39
|
|
|
40
40
|
# Save extraction results to PO files.
|
|
@@ -105,17 +105,12 @@ module Jekyll
|
|
|
105
105
|
po_files_created
|
|
106
106
|
end
|
|
107
107
|
|
|
108
|
-
#
|
|
109
|
-
#
|
|
110
|
-
#
|
|
111
|
-
# All other site-wide settings (locales_dir, extraction, translation,
|
|
112
|
-
# logging, etc.) are preserved unchanged; only the locale list is widened.
|
|
108
|
+
# The site config with every locale any page uses, not only the
|
|
109
|
+
# site-wide list (see SiteConfigAccessor.all_configured_locales).
|
|
113
110
|
#
|
|
114
111
|
# @return [PageLocalesConfig]
|
|
115
112
|
def merge_config
|
|
116
|
-
|
|
117
|
-
merged_data = (@site_config.data['with_locales_data'] || {}).merge('locales' => all_locales)
|
|
118
|
-
PageLocalesConfig.new({ 'with_locales_data' => merged_data })
|
|
113
|
+
SiteConfigAccessor.site_config(@site, all_locales: true)
|
|
119
114
|
end
|
|
120
115
|
|
|
121
116
|
private :save_po_files
|