jekyll-l10n 2.0.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +44 -0
- data/lib/jekyll-l10n/configuration/schema.rb +308 -0
- data/lib/jekyll-l10n/configuration.rb +165 -0
- data/lib/jekyll-l10n/constants.rb +45 -10
- data/lib/jekyll-l10n/errors.rb +7 -0
- data/lib/jekyll-l10n/extraction/compendium_merger.rb +9 -11
- data/lib/jekyll-l10n/extraction/compendium_translator.rb +27 -11
- data/lib/jekyll-l10n/extraction/config_loader.rb +4 -4
- data/lib/jekyll-l10n/extraction/coverage_report.rb +129 -0
- data/lib/jekyll-l10n/extraction/dom_text_extractor.rb +2 -31
- data/lib/jekyll-l10n/extraction/extractor.rb +48 -28
- data/lib/jekyll-l10n/extraction/html_string_extractor.rb +24 -10
- data/lib/jekyll-l10n/extraction/logger.rb +6 -5
- data/lib/jekyll-l10n/extraction/result_saver.rb +5 -10
- data/lib/jekyll-l10n/instrumentation.rb +172 -139
- data/lib/jekyll-l10n/jekyll/generator.rb +10 -22
- data/lib/jekyll-l10n/jekyll/localized_page.rb +27 -49
- data/lib/jekyll-l10n/jekyll/post_write_html_reprocessor.rb +42 -25
- data/lib/jekyll-l10n/jekyll/post_write_processor.rb +2 -1
- data/lib/jekyll-l10n/jekyll/regeneration_checker.rb +16 -10
- data/lib/jekyll-l10n/jekyll/url_filter.rb +8 -7
- data/lib/jekyll-l10n/po_file/entry_parser.rb +155 -0
- data/lib/jekyll-l10n/po_file/fuzzy_matcher.rb +67 -53
- data/lib/jekyll-l10n/po_file/loader.rb +20 -11
- data/lib/jekyll-l10n/po_file/manager.rb +32 -17
- data/lib/jekyll-l10n/po_file/merger.rb +2 -1
- data/lib/jekyll-l10n/po_file/path_builder.rb +11 -1
- data/lib/jekyll-l10n/po_file/reader.rb +40 -508
- data/lib/jekyll-l10n/po_file/writer.rb +31 -16
- data/lib/jekyll-l10n/translation/block_text_extractor.rb +3 -7
- data/lib/jekyll-l10n/translation/html_translator.rb +136 -157
- data/lib/jekyll-l10n/translation/libre_translate_languages.rb +73 -0
- data/lib/jekyll-l10n/translation/libre_translator.rb +134 -78
- data/lib/jekyll-l10n/translation/request_slots.rb +69 -0
- data/lib/jekyll-l10n/translation/translation_progress.rb +77 -0
- data/lib/jekyll-l10n/translation/translator.rb +20 -17
- data/lib/jekyll-l10n/utils/baseurl_normalizer.rb +108 -0
- data/lib/jekyll-l10n/utils/debug_logger.rb +4 -51
- data/lib/jekyll-l10n/utils/error_handler.rb +9 -24
- data/lib/jekyll-l10n/utils/external_link_icon_preserver.rb +5 -62
- data/lib/jekyll-l10n/utils/html_parser.rb +24 -31
- data/lib/jekyll-l10n/utils/html_text_utils.rb +146 -91
- data/lib/jekyll-l10n/utils/logger_formatter.rb +6 -4
- data/lib/jekyll-l10n/utils/msgstr_validator.rb +74 -0
- data/lib/jekyll-l10n/utils/page_locales_config.rb +126 -107
- data/lib/jekyll-l10n/utils/po_entry_converter.rb +0 -43
- data/lib/jekyll-l10n/utils/site_config_accessor.rb +56 -54
- data/lib/jekyll-l10n/utils/translation_resolver.rb +56 -6
- data/lib/jekyll-l10n/utils/url_path_builder.rb +5 -27
- data/lib/jekyll-l10n/utils/url_transformer.rb +5 -11
- data/lib/jekyll-l10n/version.rb +1 -1
- data/lib/jekyll-l10n.rb +9 -6
- metadata +12 -3
|
@@ -8,14 +8,12 @@ module Jekyll
|
|
|
8
8
|
#
|
|
9
9
|
# DebugLogger provides detailed logging of translation attempts for
|
|
10
10
|
# debugging normalization and matching issues. It logs when translations
|
|
11
|
-
# are found, when translations are missing,
|
|
12
|
-
#
|
|
11
|
+
# are found, when translations are missing, and similarity analysis for
|
|
12
|
+
# debugging.
|
|
13
13
|
#
|
|
14
14
|
# Key responsibilities:
|
|
15
15
|
# * Log translation match details
|
|
16
16
|
# * Log missing translations with similar key analysis
|
|
17
|
-
# * Log character-level differences for debugging
|
|
18
|
-
# * Provide context around matching failures
|
|
19
17
|
# * Support verbose translation debugging
|
|
20
18
|
#
|
|
21
19
|
# @see Jekyll::L10n::HtmlTranslator for translation workflow context
|
|
@@ -69,18 +67,6 @@ module Jekyll
|
|
|
69
67
|
log_alert_comparison(text, normalized_text, key_from_hash)
|
|
70
68
|
end
|
|
71
69
|
|
|
72
|
-
def self.log_first_difference(normalized_text, key_from_hash)
|
|
73
|
-
(0...normalized_text.length).each do |i|
|
|
74
|
-
next unless normalized_text[i] != key_from_hash[i]
|
|
75
|
-
|
|
76
|
-
log_difference_at_position(normalized_text, key_from_hash, i)
|
|
77
|
-
log_context_strings(normalized_text, key_from_hash, i)
|
|
78
|
-
break
|
|
79
|
-
end
|
|
80
|
-
|
|
81
|
-
log_normalization_match
|
|
82
|
-
end
|
|
83
|
-
|
|
84
70
|
def self.log_found_translation(text, translated)
|
|
85
71
|
return unless translated && translated.length > Jekyll::L10n::Constants::LOG_THRESHOLD_SHORT
|
|
86
72
|
|
|
@@ -108,46 +94,13 @@ module Jekyll
|
|
|
108
94
|
log_message("[HtmlTranslator] NORMALIZED match: #{normalized_text == key_from_hash}")
|
|
109
95
|
end
|
|
110
96
|
|
|
111
|
-
def self.log_difference_at_position(text1, text2, index)
|
|
112
|
-
char1 = text1[index]
|
|
113
|
-
char2 = text2[index]
|
|
114
|
-
log_message(
|
|
115
|
-
"[HtmlTranslator] DIFF at position #{index}: " \
|
|
116
|
-
"NORMALIZED=#{char1.inspect} (#{char1.ord}) vs KEY=#{char2.inspect} (#{char2.ord})"
|
|
117
|
-
)
|
|
118
|
-
end
|
|
119
|
-
|
|
120
|
-
def self.log_context_strings(normalized_text, key_from_hash, index)
|
|
121
|
-
normalized_context = context_string(normalized_text, index)
|
|
122
|
-
key_context = context_string(key_from_hash, index)
|
|
123
|
-
|
|
124
|
-
log_message("[HtmlTranslator] Context NORMALIZED: ...#{normalized_context.inspect}...")
|
|
125
|
-
log_message("[HtmlTranslator] Context KEY: ...#{key_context.inspect}...")
|
|
126
|
-
end
|
|
127
|
-
|
|
128
|
-
def self.log_normalization_match
|
|
129
|
-
log_message(
|
|
130
|
-
'[HtmlTranslator] ✓ Text matches after normalization - translation will be applied'
|
|
131
|
-
)
|
|
132
|
-
end
|
|
133
|
-
|
|
134
97
|
def self.log_message(message)
|
|
135
|
-
Jekyll.logger.info
|
|
136
|
-
end
|
|
137
|
-
|
|
138
|
-
def self.context_string(text, index)
|
|
139
|
-
start_idx = [0, index - Jekyll::L10n::Constants::BACKTRACE_CONTEXT_LENGTH].max
|
|
140
|
-
end_idx = [index + Jekyll::L10n::Constants::BACKTRACE_CONTEXT_LENGTH, text.length - 1].min
|
|
141
|
-
text[start_idx..end_idx]
|
|
98
|
+
Jekyll.logger.info Constants::LOG_PREFIX, message
|
|
142
99
|
end
|
|
143
100
|
|
|
144
101
|
private_class_method :log_no_translation_found,
|
|
145
102
|
:log_alert_comparison,
|
|
146
|
-
:
|
|
147
|
-
:log_context_strings,
|
|
148
|
-
:log_normalization_match,
|
|
149
|
-
:log_message,
|
|
150
|
-
:context_string
|
|
103
|
+
:log_message
|
|
151
104
|
end
|
|
152
105
|
end
|
|
153
106
|
end
|
|
@@ -1,24 +1,25 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative '../constants'
|
|
4
|
+
|
|
3
5
|
module Jekyll
|
|
4
6
|
module L10n
|
|
5
7
|
# Handles errors with graceful fallback and logging.
|
|
6
8
|
#
|
|
7
9
|
# ErrorHandler provides error handling utilities for gracefully managing
|
|
8
|
-
# exceptions
|
|
9
|
-
#
|
|
10
|
+
# exceptions and logging them with context. This keeps the build process
|
|
11
|
+
# running even when individual operations fail.
|
|
10
12
|
#
|
|
11
13
|
# Key responsibilities:
|
|
12
14
|
# * Execute code with error catching
|
|
13
15
|
# * Log errors with context information
|
|
14
|
-
# * Provide fallback default values on error
|
|
15
16
|
# * Optional debug backtraces
|
|
16
17
|
#
|
|
17
18
|
# @example
|
|
18
|
-
# result = ErrorHandler.
|
|
19
|
+
# result = ErrorHandler.handle_with_logging('file loading') do
|
|
19
20
|
# load_file('config.yml')
|
|
20
21
|
# end
|
|
21
|
-
# # Returns loaded data or
|
|
22
|
+
# # Returns loaded data or nil on error
|
|
22
23
|
class ErrorHandler
|
|
23
24
|
# Execute code with error catching and fallback logging.
|
|
24
25
|
#
|
|
@@ -35,22 +36,6 @@ module Jekyll
|
|
|
35
36
|
nil
|
|
36
37
|
end
|
|
37
38
|
|
|
38
|
-
# Execute code with error catching and fallback value.
|
|
39
|
-
#
|
|
40
|
-
# Executes the provided block and catches any StandardError, logging it
|
|
41
|
-
# with context and returning the default value.
|
|
42
|
-
#
|
|
43
|
-
# @param context [String] Context describing what was being done
|
|
44
|
-
# @param default_value [Object] Default value to return on error
|
|
45
|
-
# @yield Code to execute
|
|
46
|
-
# @return [Object] Result of block or default_value on error
|
|
47
|
-
def self.handle_with_default(context, default_value)
|
|
48
|
-
yield
|
|
49
|
-
rescue StandardError => e
|
|
50
|
-
log_error(context, e)
|
|
51
|
-
default_value
|
|
52
|
-
end
|
|
53
|
-
|
|
54
39
|
# Log an error with context.
|
|
55
40
|
#
|
|
56
41
|
# Logs error message and optionally backtrace if DEBUG environment variable is set.
|
|
@@ -59,8 +44,8 @@ module Jekyll
|
|
|
59
44
|
# @param error [StandardError] The error that occurred
|
|
60
45
|
# @return [void]
|
|
61
46
|
def self.log_error(context, error)
|
|
62
|
-
Jekyll.logger.error
|
|
63
|
-
Jekyll.logger.debug
|
|
47
|
+
Jekyll.logger.error Constants::LOG_PREFIX, "Error in #{context}: #{error.message}"
|
|
48
|
+
Jekyll.logger.debug Constants::LOG_PREFIX, error.backtrace.join("\n") if ENV['DEBUG']
|
|
64
49
|
end
|
|
65
50
|
|
|
66
51
|
# Log a recoverable warning with context.
|
|
@@ -72,7 +57,7 @@ module Jekyll
|
|
|
72
57
|
# @param error [StandardError] The error that occurred
|
|
73
58
|
# @return [void]
|
|
74
59
|
def self.log_warning(context, error)
|
|
75
|
-
Jekyll.logger.warn
|
|
60
|
+
Jekyll.logger.warn Constants::LOG_PREFIX, "Error in #{context}: #{error.message}"
|
|
76
61
|
end
|
|
77
62
|
end
|
|
78
63
|
end
|
|
@@ -23,12 +23,10 @@ module Jekyll
|
|
|
23
23
|
# @param translated_html [String] Translated HTML potentially missing icons
|
|
24
24
|
# @return [String] Translated HTML with icons restored
|
|
25
25
|
def self.preserve(original_html, translated_html)
|
|
26
|
-
#
|
|
27
|
-
#
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
original_doc = Nokogiri::HTML(original_html)
|
|
31
|
-
translated_doc = Nokogiri::HTML(translated_html)
|
|
26
|
+
# Parse as documents, not fragments, to keep the doctype and the html,
|
|
27
|
+
# head and body elements.
|
|
28
|
+
original_doc = HtmlParser.parse_document(original_html)
|
|
29
|
+
translated_doc = HtmlParser.parse_document(translated_html)
|
|
32
30
|
|
|
33
31
|
# Find all external link icons in original
|
|
34
32
|
original_icons = original_doc.css('i.fa-external-link')
|
|
@@ -37,10 +35,7 @@ module Jekyll
|
|
|
37
35
|
# Process each icon found in the original HTML
|
|
38
36
|
restore_icons_to_translated(original_icons, translated_doc)
|
|
39
37
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
# Remove the auto-inserted meta tag by libxml2 during HTML serialization
|
|
43
|
-
HtmlParser.remove_meta_charset(result)
|
|
38
|
+
translated_doc.to_html
|
|
44
39
|
rescue StandardError => e
|
|
45
40
|
ErrorHandler.log_error('preserving external link icons', e)
|
|
46
41
|
translated_html
|
|
@@ -66,58 +61,6 @@ module Jekyll
|
|
|
66
61
|
translated_link.add_child(icon_copy)
|
|
67
62
|
end
|
|
68
63
|
end
|
|
69
|
-
|
|
70
|
-
# Alternative method that preserves ALL inline elements, not just icons
|
|
71
|
-
# Useful for preserving SVG icons, badges, etc.
|
|
72
|
-
#
|
|
73
|
-
# @param original_html [String] Original HTML
|
|
74
|
-
# @param translated_html [String] Translated HTML
|
|
75
|
-
# @return [String] Translated HTML with all inline elements preserved
|
|
76
|
-
def self.preserve_all_inline_elements(original_html, translated_html)
|
|
77
|
-
original_doc = Nokogiri::HTML(original_html)
|
|
78
|
-
translated_doc = Nokogiri::HTML(translated_html)
|
|
79
|
-
|
|
80
|
-
# Find all links with child elements in original
|
|
81
|
-
original_links = original_doc.css('a[href]')
|
|
82
|
-
original_links.each do |original_link|
|
|
83
|
-
preserve_link_inline_elements(original_link, translated_doc)
|
|
84
|
-
end
|
|
85
|
-
|
|
86
|
-
result = translated_doc.to_html
|
|
87
|
-
HtmlParser.remove_meta_charset(result)
|
|
88
|
-
rescue StandardError => e
|
|
89
|
-
ErrorHandler.log_error('preserving inline elements', e)
|
|
90
|
-
translated_html
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
def self.preserve_link_inline_elements(original_link, translated_doc)
|
|
94
|
-
href = original_link['href']
|
|
95
|
-
return unless href
|
|
96
|
-
|
|
97
|
-
# Get all non-text children (i, svg, span, etc.)
|
|
98
|
-
inline_children = original_link.children.select(&:element?)
|
|
99
|
-
return if inline_children.empty?
|
|
100
|
-
|
|
101
|
-
# Find matching link in translated version
|
|
102
|
-
translated_link = translated_doc.css("a[href=\"#{href}\"]").first
|
|
103
|
-
return unless translated_link
|
|
104
|
-
|
|
105
|
-
# Append inline elements that don't already exist
|
|
106
|
-
inline_children.each do |child|
|
|
107
|
-
add_child_if_not_exists(child, translated_link)
|
|
108
|
-
end
|
|
109
|
-
end
|
|
110
|
-
|
|
111
|
-
def self.add_child_if_not_exists(child, translated_link)
|
|
112
|
-
child_class = child['class'] || ''
|
|
113
|
-
# Check if similar element already exists
|
|
114
|
-
existing = translated_link.css("#{child.name}.#{child_class.split.first}").first
|
|
115
|
-
return if existing
|
|
116
|
-
|
|
117
|
-
# Add the element
|
|
118
|
-
child_copy = child.dup
|
|
119
|
-
translated_link.add_child(child_copy)
|
|
120
|
-
end
|
|
121
64
|
end
|
|
122
65
|
end
|
|
123
66
|
end
|
|
@@ -1,51 +1,44 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'nokogiri'
|
|
4
|
+
|
|
3
5
|
module Jekyll
|
|
4
6
|
module L10n
|
|
5
|
-
# Parses HTML
|
|
7
|
+
# Parses HTML with Nokogiri's HTML5 parser.
|
|
6
8
|
#
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
#
|
|
9
|
+
# Every part of the plugin parses through here, so extraction and
|
|
10
|
+
# translation see the same tree and produce the same msgids. The HTML5
|
|
11
|
+
# parser keeps a page's doctype as written and, unlike libxml2's HTML4
|
|
12
|
+
# serializer, never adds or rewrites a Content-Type meta tag, so a parsed
|
|
13
|
+
# page is written back with `to_html` as is.
|
|
11
14
|
#
|
|
12
15
|
# Key responsibilities:
|
|
13
|
-
# * Parse full HTML documents
|
|
14
|
-
# * Parse HTML fragments
|
|
15
|
-
# * Remove auto-inserted meta charset tags
|
|
16
|
+
# * Parse full HTML documents
|
|
17
|
+
# * Parse HTML fragments, such as a msgstr
|
|
16
18
|
class HtmlParser
|
|
19
|
+
# No limit on nesting depth or attributes per element: the parser's
|
|
20
|
+
# defaults raise on large generated pages.
|
|
21
|
+
PARSE_OPTIONS = { max_tree_depth: -1, max_attributes: -1 }.freeze
|
|
22
|
+
|
|
17
23
|
# Parse HTML as a full document.
|
|
18
24
|
#
|
|
19
|
-
# Preserves DOCTYPE, html tag, and document structure. Use this for complete
|
|
20
|
-
# HTML documents. Auto-inserted meta tags can be removed with remove_meta_charset.
|
|
21
|
-
#
|
|
22
25
|
# @param html [String] HTML content to parse
|
|
23
|
-
# @return [Nokogiri::
|
|
26
|
+
# @return [Nokogiri::HTML5::Document] Parsed HTML document
|
|
24
27
|
def self.parse_document(html)
|
|
25
|
-
Nokogiri::
|
|
28
|
+
Nokogiri::HTML5(html, **PARSE_OPTIONS)
|
|
26
29
|
end
|
|
27
30
|
|
|
28
31
|
# Parse HTML as a fragment.
|
|
29
32
|
#
|
|
30
|
-
#
|
|
31
|
-
#
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
# @return [Nokogiri::
|
|
35
|
-
def self.parse_fragment(html)
|
|
36
|
-
Nokogiri::
|
|
37
|
-
end
|
|
33
|
+
# @param html [String] HTML content to parse
|
|
34
|
+
# @param context [Nokogiri::XML::Element, nil] (keyword) Element the
|
|
35
|
+
# fragment will be inserted into, which decides how it parses (inside
|
|
36
|
+
# a <table> or an <svg>, for example); nil parses it on its own
|
|
37
|
+
# @return [Nokogiri::HTML5::DocumentFragment] Parsed fragment
|
|
38
|
+
def self.parse_fragment(html, context: nil)
|
|
39
|
+
return Nokogiri::HTML5.fragment(html, **PARSE_OPTIONS) unless context
|
|
38
40
|
|
|
39
|
-
|
|
40
|
-
#
|
|
41
|
-
# Nokogiri/libxml2 automatically inserts a meta charset tag during
|
|
42
|
-
# serialization. This removes that tag which was not in the original HTML.
|
|
43
|
-
#
|
|
44
|
-
# @param html_string [String] Serialized HTML
|
|
45
|
-
# @return [String] HTML with meta charset tag removed
|
|
46
|
-
def self.remove_meta_charset(html_string)
|
|
47
|
-
pattern = %r{<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">\n?}
|
|
48
|
-
html_string.gsub(pattern, '')
|
|
41
|
+
Nokogiri::HTML5::DocumentFragment.new(context.document, html, context, **PARSE_OPTIONS)
|
|
49
42
|
end
|
|
50
43
|
end
|
|
51
44
|
end
|
|
@@ -9,16 +9,15 @@ module Jekyll
|
|
|
9
9
|
# Utilities for extracting and manipulating HTML text content.
|
|
10
10
|
#
|
|
11
11
|
# HtmlTextUtils provides helpers for extracting text from HTML elements while
|
|
12
|
-
# preserving inline formatting, removing block-level elements,
|
|
13
|
-
#
|
|
14
|
-
#
|
|
12
|
+
# preserving inline formatting, removing block-level elements, and cleaning
|
|
13
|
+
# up icon tags. These utilities support the extraction and translation
|
|
14
|
+
# pipelines.
|
|
15
15
|
#
|
|
16
16
|
# Key responsibilities:
|
|
17
17
|
# * Extract text with inline HTML tags preserved
|
|
18
18
|
# * Remove block-level elements from cloned nodes
|
|
19
19
|
# * Remove empty icon tags
|
|
20
|
-
# *
|
|
21
|
-
# * Validate extracted text content
|
|
20
|
+
# * Tell content elements apart from other nodes
|
|
22
21
|
module HtmlTextUtils
|
|
23
22
|
# Extended content elements for text extraction (includes inline elements)
|
|
24
23
|
CONTENT_ELEMENTS = %w[
|
|
@@ -31,24 +30,6 @@ module Jekyll
|
|
|
31
30
|
CONTAINER_ELEMENTS = HtmlElements::CONTAINER_ELEMENTS
|
|
32
31
|
ALL_BLOCK_ELEMENTS = (CONTENT_ELEMENTS + CONTAINER_ELEMENTS).freeze
|
|
33
32
|
|
|
34
|
-
# Decode HTML entities to plain text.
|
|
35
|
-
#
|
|
36
|
-
# Converts HTML entities (&, <, etc.) to their plain text equivalents.
|
|
37
|
-
# Uses CGI.unescape_html if available, falls back to manual replacement.
|
|
38
|
-
#
|
|
39
|
-
# @param text [String] Text with HTML entities
|
|
40
|
-
# @return [String] Text with entities decoded
|
|
41
|
-
def self.decode_html_entities(text)
|
|
42
|
-
require 'cgi'
|
|
43
|
-
CGI.unescape_html(text)
|
|
44
|
-
rescue StandardError
|
|
45
|
-
text.gsub('&', '&')
|
|
46
|
-
.gsub('<', '<')
|
|
47
|
-
.gsub('>', '>')
|
|
48
|
-
.gsub('"', '"')
|
|
49
|
-
.gsub(''', "'")
|
|
50
|
-
end
|
|
51
|
-
|
|
52
33
|
# Remove preformatted code blocks from a node.
|
|
53
34
|
#
|
|
54
35
|
# Removes all <pre> elements entirely. With highlighter: none in Jekyll config,
|
|
@@ -73,22 +54,11 @@ module Jekyll
|
|
|
73
54
|
# @param node [Nokogiri::XML::Node] Node to process (modified in place)
|
|
74
55
|
# @return [void]
|
|
75
56
|
def self.remove_block_elements_from_node(node)
|
|
76
|
-
|
|
77
|
-
node.xpath(".//#{tag}").each do |elem|
|
|
78
|
-
elem.replace(elem.children)
|
|
79
|
-
end
|
|
80
|
-
end
|
|
57
|
+
node.css(BLOCK_SELECTOR).each { |elem| elem.replace(elem.children) }
|
|
81
58
|
end
|
|
82
59
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
# Alias for remove_block_elements_from_node for convenience.
|
|
86
|
-
#
|
|
87
|
-
# @param node [Nokogiri::XML::Node] Node to process
|
|
88
|
-
# @return [void]
|
|
89
|
-
def self.remove_block_elements(node)
|
|
90
|
-
remove_block_elements_from_node(node)
|
|
91
|
-
end
|
|
60
|
+
BLOCK_SELECTOR = HtmlElements::BLOCK_ELEMENTS.join(',').freeze
|
|
61
|
+
private_constant :BLOCK_SELECTOR
|
|
92
62
|
|
|
93
63
|
# Remove empty icon tags from a node.
|
|
94
64
|
#
|
|
@@ -109,78 +79,177 @@ module Jekyll
|
|
|
109
79
|
# table rows, etc.) and must never be attempted.
|
|
110
80
|
LAYOUT_ONLY_ELEMENTS = (HtmlElements::BLOCK_ELEMENTS - ALL_BLOCK_ELEMENTS - %w[pre]).freeze
|
|
111
81
|
|
|
112
|
-
# Inline elements whose content is translatable — replaced with <g id="N"
|
|
113
|
-
|
|
82
|
+
# Inline elements whose content is translatable — replaced with <g id="N">,
|
|
83
|
+
# nested ones included, so their attributes never reach a msgid.
|
|
84
|
+
PLACEHOLDERED_INLINE_TAGS = %w[
|
|
85
|
+
a span em strong b u i abbr mark label
|
|
86
|
+
sup sub small s del ins q cite dfn time bdi bdo
|
|
87
|
+
].freeze
|
|
88
|
+
|
|
89
|
+
# Inline elements without translatable content — replaced with <x id="N"></x>.
|
|
90
|
+
# The closing tag is explicit because HTML parsers ignore "/>" on unknown tags.
|
|
91
|
+
EMPTY_INLINE_TAGS = %w[img br wbr input svg].freeze
|
|
114
92
|
|
|
115
|
-
# Inline elements with literal content — tag kept,
|
|
93
|
+
# Inline elements with literal content — tag and content kept, attributes
|
|
94
|
+
# stripped.
|
|
116
95
|
LITERAL_INLINE_TAGS = %w[code var kbd samp].freeze
|
|
117
96
|
|
|
118
|
-
#
|
|
119
|
-
|
|
97
|
+
# The markup removed from an element's text, for putting it back.
|
|
98
|
+
#
|
|
99
|
+
# text: the msgid, with <g id="N"> and <x id="N"></x> placeholders
|
|
100
|
+
# placeholders: id => [tag, { attribute => value }] of the element each
|
|
101
|
+
# placeholder stands for
|
|
102
|
+
# literals: [tag, { attribute => value }] of each literal element, in order
|
|
103
|
+
InlineMarkup = Struct.new(:text, :placeholders, :literals)
|
|
120
104
|
|
|
121
105
|
# Extract text with inline tags preserved.
|
|
122
106
|
#
|
|
123
107
|
# Extracts text from an element, removes block elements and empty icons,
|
|
124
|
-
# replaces
|
|
125
|
-
#
|
|
126
|
-
#
|
|
108
|
+
# replaces inline elements with placeholders, and strips attributes from
|
|
109
|
+
# literal elements (<code> etc.). HTML entities (e.g. <, >) are
|
|
110
|
+
# preserved verbatim.
|
|
127
111
|
#
|
|
128
112
|
# @param node [Nokogiri::XML::Node] Element to extract from
|
|
129
|
-
# @return [String] Extracted and normalized text
|
|
113
|
+
# @return [String, nil] Extracted and normalized text
|
|
130
114
|
def self.extract_with_inline_tags(node)
|
|
115
|
+
inline_markup(node).text
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Extract text like {.extract_with_inline_tags}, and record the markup each
|
|
119
|
+
# placeholder and literal element stands for.
|
|
120
|
+
#
|
|
121
|
+
# Extraction and translation both number placeholders here, so a msgstr's
|
|
122
|
+
# ids always refer to the same elements of the page.
|
|
123
|
+
#
|
|
124
|
+
# @param node [Nokogiri::XML::Node] Element to extract from
|
|
125
|
+
# @return [InlineMarkup]
|
|
126
|
+
def self.inline_markup(node)
|
|
131
127
|
clone = node.dup
|
|
132
128
|
remove_code_blocks(clone)
|
|
133
129
|
remove_block_elements_from_node(clone)
|
|
134
130
|
remove_empty_icon_tags(clone)
|
|
135
|
-
|
|
131
|
+
placeholders = replace_inline_elements_with_placeholders(clone)
|
|
132
|
+
literals = strip_attributes_from_literal_elements(clone)
|
|
133
|
+
replace_non_breaking_spaces(clone)
|
|
136
134
|
|
|
137
|
-
|
|
138
|
-
text&.then { |t| TextNormalizer.normalize(t).strip }
|
|
135
|
+
InlineMarkup.new(TextNormalizer.normalize(clone.inner_html)&.strip, placeholders, literals)
|
|
139
136
|
end
|
|
140
137
|
|
|
141
|
-
# Replace
|
|
138
|
+
# Replace inline elements with numbered placeholders.
|
|
142
139
|
#
|
|
143
|
-
#
|
|
144
|
-
#
|
|
145
|
-
#
|
|
146
|
-
#
|
|
140
|
+
# Elements are numbered in document order, depth first; an element nested
|
|
141
|
+
# in another gets its own placeholder inside the outer one. Elements with
|
|
142
|
+
# no text and no empty inline element inside are dropped. Elements inside a
|
|
143
|
+
# literal element are part of its literal content and are left alone.
|
|
147
144
|
#
|
|
148
145
|
# @param node [Nokogiri::XML::Node] Node to process (modified in place)
|
|
149
|
-
# @return [
|
|
150
|
-
def self.
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
g['id'] = g_index.to_s
|
|
159
|
-
el.children.each { |child| g.add_child(child.dup) }
|
|
160
|
-
el.replace(g)
|
|
161
|
-
end
|
|
146
|
+
# @return [Hash{String => Array}] id => [tag, attributes] of each replaced element
|
|
147
|
+
def self.replace_inline_elements_with_placeholders(node)
|
|
148
|
+
placeholders = {}
|
|
149
|
+
placeholder_candidates(node).each do |el|
|
|
150
|
+
next el.remove if droppable?(el)
|
|
151
|
+
|
|
152
|
+
id = (placeholders.size + 1).to_s
|
|
153
|
+
placeholders[id] = [el.name, attributes_of(el)]
|
|
154
|
+
el.replace(placeholder_for(el, id))
|
|
162
155
|
end
|
|
163
|
-
|
|
156
|
+
placeholders
|
|
164
157
|
end
|
|
165
158
|
|
|
166
|
-
def self.
|
|
167
|
-
node.css(PLACEHOLDERED_INLINE_TAGS.join(',')).reject do |el|
|
|
168
|
-
el.ancestors.any?
|
|
169
|
-
PLACEHOLDERED_INLINE_TAGS.include?(a.name) || LITERAL_INLINE_TAGS.include?(a.name)
|
|
170
|
-
end
|
|
159
|
+
def self.placeholder_candidates(node)
|
|
160
|
+
node.css((PLACEHOLDERED_INLINE_TAGS + EMPTY_INLINE_TAGS).join(',')).reject do |el|
|
|
161
|
+
el.ancestors.any? { |a| LITERAL_INLINE_TAGS.include?(a.name) }
|
|
171
162
|
end
|
|
172
163
|
end
|
|
173
164
|
|
|
174
|
-
|
|
165
|
+
def self.droppable?(element)
|
|
166
|
+
PLACEHOLDERED_INLINE_TAGS.include?(element.name) && element.text.strip.empty? &&
|
|
167
|
+
element.css(EMPTY_INLINE_TAGS.join(',')).empty?
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def self.placeholder_for(element, id)
|
|
171
|
+
tag = EMPTY_INLINE_TAGS.include?(element.name) ? 'x' : 'g'
|
|
172
|
+
placeholder = element.document.create_element(tag)
|
|
173
|
+
placeholder['id'] = id
|
|
174
|
+
# Move, not copy: nested elements still waiting in the candidate list
|
|
175
|
+
# must stay the nodes that list holds.
|
|
176
|
+
placeholder.children = element.children unless tag == 'x'
|
|
177
|
+
placeholder
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def self.attributes_of(element)
|
|
181
|
+
element.attribute_nodes.to_h { |attr| [attr.name, attr.value] }
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# Strip all attributes from literal-content inline elements.
|
|
175
185
|
#
|
|
176
186
|
# @param node [Nokogiri::XML::Node] Node to process (modified in place)
|
|
177
|
-
# @return [
|
|
187
|
+
# @return [Array<Array>] [tag, attributes] of each literal element, in order
|
|
178
188
|
def self.strip_attributes_from_literal_elements(node)
|
|
179
|
-
node.css(LITERAL_INLINE_TAGS.join(',')).
|
|
180
|
-
|
|
189
|
+
node.css(LITERAL_INLINE_TAGS.join(',')).map do |el|
|
|
190
|
+
attrs = attributes_of(el)
|
|
191
|
+
attrs.each_key { |name| el.remove_attribute(name) }
|
|
192
|
+
[el.name, attrs]
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# Turn non-breaking spaces into plain ones, in text and in attribute
|
|
197
|
+
# values. The HTML5 serializer writes U+00A0 as " ", which
|
|
198
|
+
# TextNormalizer would not collapse; msgids keep a plain space instead.
|
|
199
|
+
#
|
|
200
|
+
# @param node [Nokogiri::XML::Node] Node to process (modified in place)
|
|
201
|
+
# @return [void]
|
|
202
|
+
def self.replace_non_breaking_spaces(node)
|
|
203
|
+
node.traverse do |child|
|
|
204
|
+
if child.text?
|
|
205
|
+
child.content = child.content.tr(NBSP, ' ') if child.content.include?(NBSP)
|
|
206
|
+
elsif child.element?
|
|
207
|
+
child.attribute_nodes.each { |attr| attr.value = attr.value.tr(NBSP, ' ') if attr.value.include?(NBSP) }
|
|
208
|
+
end
|
|
181
209
|
end
|
|
182
210
|
end
|
|
183
211
|
|
|
212
|
+
NBSP = "\u00A0"
|
|
213
|
+
private_constant :NBSP
|
|
214
|
+
|
|
215
|
+
private_class_method :placeholder_candidates, :droppable?, :placeholder_for, :attributes_of
|
|
216
|
+
|
|
217
|
+
# The msgid of an element's text, or nil when it has none: its text sits
|
|
218
|
+
# only in block children, a layout-only block child would be flattened, or
|
|
219
|
+
# the text is too short to translate.
|
|
220
|
+
#
|
|
221
|
+
# Extraction and translation both decide here, so translation looks a
|
|
222
|
+
# block up only by a msgid extraction gave it.
|
|
223
|
+
#
|
|
224
|
+
# @param node [Nokogiri::XML::Element] Element to read
|
|
225
|
+
# @return [String, nil] The msgid, or nil
|
|
226
|
+
def self.block_text(node)
|
|
227
|
+
return nil if only_block_children?(node) || layout_block_children?(node)
|
|
228
|
+
|
|
229
|
+
text = extract_with_inline_tags(node)
|
|
230
|
+
TextValidator.valid?(text) ? text : nil
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
# Check whether a node's text sits only in block children: it has a block
|
|
234
|
+
# child and no text or inline element of its own.
|
|
235
|
+
#
|
|
236
|
+
# @param node [Nokogiri::XML::Node] Node to inspect
|
|
237
|
+
# @return [Boolean]
|
|
238
|
+
def self.only_block_children?(node)
|
|
239
|
+
node.children.none? { |child| own_content?(child) } &&
|
|
240
|
+
node.children.any? { |child| block_element?(child) }
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
# A non-blank text node, or an element outside ALL_BLOCK_ELEMENTS.
|
|
244
|
+
def self.own_content?(child)
|
|
245
|
+
(child.text? && !child.content.strip.empty?) || (child.element? && !block_element?(child))
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def self.block_element?(child)
|
|
249
|
+
child.element? && ALL_BLOCK_ELEMENTS.include?(child.name)
|
|
250
|
+
end
|
|
251
|
+
private_class_method :own_content?, :block_element?
|
|
252
|
+
|
|
184
253
|
# Check if a node has any direct child that is a layout-only block element.
|
|
185
254
|
#
|
|
186
255
|
# Layout-only elements (ul, ol, dl, table, form, etc.) are in
|
|
@@ -195,20 +264,6 @@ module Jekyll
|
|
|
195
264
|
node.children.any? { |c| c.element? && LAYOUT_ONLY_ELEMENTS.include?(c.name) }
|
|
196
265
|
end
|
|
197
266
|
|
|
198
|
-
# Extract and validate text from a node.
|
|
199
|
-
#
|
|
200
|
-
# Extracts text from element if it's a content element, then validates it
|
|
201
|
-
# meets minimum length requirements.
|
|
202
|
-
#
|
|
203
|
-
# @param node [Nokogiri::XML::Node] Node to extract from
|
|
204
|
-
# @return [String, nil] Validated text, or nil if not extractable or invalid
|
|
205
|
-
def self.extract_and_validate_text(node)
|
|
206
|
-
return nil unless extractable?(node)
|
|
207
|
-
|
|
208
|
-
text = extract_with_inline_tags(node)
|
|
209
|
-
TextValidator.valid?(text) ? text : nil
|
|
210
|
-
end
|
|
211
|
-
|
|
212
267
|
# Check if a node is extractable (content element).
|
|
213
268
|
#
|
|
214
269
|
# @param node [Nokogiri::XML::Node] Node to check
|