html2rss 0.32.4 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -2
- data/lib/html2rss/article/deduplicator.rb +5 -2
- data/lib/html2rss/article/enclosure.rb +185 -25
- data/lib/html2rss/article/title.rb +203 -0
- data/lib/html2rss/article.rb +79 -13
- data/lib/html2rss/auto_source/README.md +16 -8
- data/lib/html2rss/auto_source/cleanup.rb +127 -53
- data/lib/html2rss/auto_source/corroboration.rb +167 -0
- data/lib/html2rss/auto_source/page_links/anchor_claims.rb +82 -0
- data/lib/html2rss/auto_source/page_links/card_units.rb +77 -0
- data/lib/html2rss/auto_source/page_links/label_binding.rb +79 -0
- data/lib/html2rss/auto_source/page_links/sponsored_cards.rb +147 -0
- data/lib/html2rss/auto_source/page_links.rb +164 -0
- data/lib/html2rss/auto_source/scraper/html.rb +11 -45
- data/lib/html2rss/auto_source/scraper/json_document.rb +3 -6
- data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +202 -125
- data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +27 -38
- data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +123 -123
- data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +27 -35
- data/lib/html2rss/auto_source/scraper/json_state.rb +4 -9
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +27 -5
- data/lib/html2rss/auto_source/scraper/microdata.rb +233 -248
- data/lib/html2rss/auto_source/scraper/microformats2.rb +10 -4
- data/lib/html2rss/auto_source/scraper/native_feed.rb +28 -18
- data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +7 -2
- data/lib/html2rss/auto_source/scraper/schema/item_list.rb +5 -3
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +140 -22
- data/lib/html2rss/auto_source/scraper/schema.rb +52 -8
- data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb +3 -9
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +12 -20
- data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +1 -4
- data/lib/html2rss/auto_source/scraper/sitemap.rb +7 -14
- data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +2 -2
- data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb +1 -3
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +7 -7
- data/lib/html2rss/auto_source/scraper/xhr_articles.rb +2 -4
- data/lib/html2rss/auto_source/scraper.rb +10 -13
- data/lib/html2rss/auto_source/segment.rb +19 -5
- data/lib/html2rss/auto_source/segmenter/cluster.rb +18 -22
- data/lib/html2rss/auto_source/segmenter/list.rb +35 -59
- data/lib/html2rss/auto_source/segmenter/primary_link.rb +29 -4
- data/lib/html2rss/auto_source/segmenter/semantic.rb +23 -19
- data/lib/html2rss/auto_source/segmenter.rb +4 -8
- data/lib/html2rss/auto_source.rb +81 -26
- data/lib/html2rss/batch.rb +24 -32
- data/lib/html2rss/capture/README.md +6 -3
- data/lib/html2rss/capture.rb +51 -59
- data/lib/html2rss/channel.rb +30 -12
- data/lib/html2rss/cli/render.rb +97 -96
- data/lib/html2rss/cli/validate.rb +2 -4
- data/lib/html2rss/cli.rb +55 -38
- data/lib/html2rss/config/auto_source_contract.rb +52 -45
- data/lib/html2rss/config/issue_mapper.rb +4 -1
- data/lib/html2rss/config/request_controls.rb +16 -5
- data/lib/html2rss/config/schema.rb +23 -15
- data/lib/html2rss/config/selectors_validator.rb +3 -2
- data/lib/html2rss/config/validation_issue.rb +4 -1
- data/lib/html2rss/config/validator.rb +14 -3
- data/lib/html2rss/config.rb +54 -28
- data/lib/html2rss/contract_error.rb +9 -0
- data/lib/html2rss/defaults.rb +25 -38
- data/lib/html2rss/doctor/botasaurus.rb +18 -22
- data/lib/html2rss/error.rb +17 -3
- data/lib/html2rss/error_kind.rb +39 -0
- data/lib/html2rss/feed_builder/item_presentation.rb +22 -9
- data/lib/html2rss/feed_builder/json_feed/item.rb +2 -1
- data/lib/html2rss/feed_builder/json_feed.rb +2 -2
- data/lib/html2rss/feed_builder/rss/stylesheet.rb +3 -3
- data/lib/html2rss/feed_builder/rss.rb +50 -7
- data/lib/html2rss/feed_builder.rb +2 -1
- data/lib/html2rss/feed_pipeline/README.md +1 -1
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +16 -34
- data/lib/html2rss/feed_pipeline/strategy_plan.rb +3 -3
- data/lib/html2rss/feed_pipeline.rb +40 -27
- data/lib/html2rss/feed_resolution/README.md +1 -1
- data/lib/html2rss/feed_resolution/candidate_generator.rb +6 -8
- data/lib/html2rss/feed_resolution/options.rb +3 -7
- data/lib/html2rss/feed_resolution/policy.rb +29 -33
- data/lib/html2rss/feed_resolution/probe.rb +6 -4
- data/lib/html2rss/feed_resolution/scorer.rb +29 -29
- data/lib/html2rss/feed_resolution.rb +16 -21
- data/lib/html2rss/feed_result.rb +11 -6
- data/lib/html2rss/hash_util.rb +91 -90
- data/lib/html2rss/html/article_extractor/category_extractor.rb +7 -10
- data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
- data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +12 -33
- data/lib/html2rss/html/article_extractor/heading_extractor.rb +2 -3
- data/lib/html2rss/html/article_extractor/id_generator.rb +7 -10
- data/lib/html2rss/html/article_extractor/image_extractor.rb +9 -1
- data/lib/html2rss/html/article_extractor.rb +19 -47
- data/lib/html2rss/html/article_rules/category.rb +3 -3
- data/lib/html2rss/html/article_rules/description.rb +6 -15
- data/lib/html2rss/html/article_rules/enclosure.rb +70 -42
- data/lib/html2rss/html/article_rules/image.rb +30 -16
- data/lib/html2rss/html/feed_link.rb +7 -9
- data/lib/html2rss/html/navigator/text_extractor.rb +26 -16
- data/lib/html2rss/html/navigator.rb +39 -9
- data/lib/html2rss/html/probe.rb +55 -51
- data/lib/html2rss/html/rendering/description_builder.rb +46 -32
- data/lib/html2rss/html/rendering.rb +2 -9
- data/lib/html2rss/html/sst_article_extractor.rb +55 -145
- data/lib/html2rss/html/title_pieces/kicker.rb +147 -0
- data/lib/html2rss/html/title_pieces/nodes.rb +107 -0
- data/lib/html2rss/html/title_pieces/nokogiri_nodes.rb +70 -0
- data/lib/html2rss/html/title_pieces/sst_nodes.rb +71 -0
- data/lib/html2rss/html/title_pieces.rb +169 -0
- data/lib/html2rss/html/visibility.rb +56 -0
- data/lib/html2rss/html.rb +2 -0
- data/lib/html2rss/link_destination/destination_facts.rb +17 -1
- data/lib/html2rss/link_destination/noise_policy.rb +11 -10
- data/lib/html2rss/link_destination/path_classifier.rb +111 -42
- data/lib/html2rss/link_destination/sponsorship/label_text.rb +82 -0
- data/lib/html2rss/link_destination/sponsorship.rb +184 -0
- data/lib/html2rss/link_destination/text_classifier.rb +104 -8
- data/lib/html2rss/mcp/README.md +13 -1
- data/lib/html2rss/mcp/config_argument.rb +14 -4
- data/lib/html2rss/mcp/contract.rb +20 -9
- data/lib/html2rss/mcp/outcome.rb +18 -17
- data/lib/html2rss/mcp/runtime.rb +53 -23
- data/lib/html2rss/mcp/server/tools.rb +9 -9
- data/lib/html2rss/mcp/server.rb +13 -4
- data/lib/html2rss/option.rb +48 -0
- data/lib/html2rss/page_recon/README.md +1 -1
- data/lib/html2rss/page_recon/diagnostics.rb +9 -9
- data/lib/html2rss/page_recon.rb +0 -4
- data/lib/html2rss/recon.rb +19 -23
- data/lib/html2rss/request_service/blocked_surface.rb +5 -8
- data/lib/html2rss/request_service/botasaurus_contract.rb +25 -3
- data/lib/html2rss/request_service/botasaurus_strategy.rb +44 -12
- data/lib/html2rss/request_service/budget.rb +3 -24
- data/lib/html2rss/request_service/compressed_body.rb +36 -48
- data/lib/html2rss/request_service/context.rb +16 -17
- data/lib/html2rss/request_service/cookie_jar.rb +89 -0
- data/lib/html2rss/request_service/cookie_store.rb +79 -0
- data/lib/html2rss/request_service/feed_advertisement.rb +72 -0
- data/lib/html2rss/request_service/httpx_strategy.rb +7 -3
- data/lib/html2rss/request_service/local_file_strategy.rb +2 -3
- data/lib/html2rss/request_service/network_guard.rb +3 -2
- data/lib/html2rss/request_service/policy.rb +55 -36
- data/lib/html2rss/request_service/replay_breaker.rb +56 -0
- data/lib/html2rss/request_service/response.rb +33 -19
- data/lib/html2rss/request_service/response_cache.rb +109 -0
- data/lib/html2rss/request_service/response_caching.rb +95 -0
- data/lib/html2rss/request_service/strategy.rb +5 -11
- data/lib/html2rss/request_service.rb +71 -12
- data/lib/html2rss/request_session/pager/base.rb +3 -10
- data/lib/html2rss/request_session/pager/custom_selector.rb +4 -4
- data/lib/html2rss/request_session/pager/offset.rb +1 -7
- data/lib/html2rss/request_session/pager/rel_next.rb +1 -1
- data/lib/html2rss/request_session/pager/url_template.rb +1 -7
- data/lib/html2rss/request_session/pager.rb +1 -1
- data/lib/html2rss/request_session.rb +6 -2
- data/lib/html2rss/scoring/cluster_scorer.rb +1 -1
- data/lib/html2rss/scoring/container_assessor.rb +12 -13
- data/lib/html2rss/scoring/engine.rb +10 -38
- data/lib/html2rss/scoring/link_resolver.rb +2 -7
- data/lib/html2rss/scoring/observation.rb +2 -2
- data/lib/html2rss/selector_candidates.rb +19 -8
- data/lib/html2rss/selectors/README.md +1 -1
- data/lib/html2rss/selectors/extractors/href.rb +2 -4
- data/lib/html2rss/selectors/json_xml.rb +33 -34
- data/lib/html2rss/selectors/option_spec.rb +4 -1
- data/lib/html2rss/selectors/post_processors/base.rb +0 -8
- data/lib/html2rss/selectors/post_processors/gsub.rb +9 -10
- data/lib/html2rss/selectors/post_processors/html_transformers/keep_playable_media.rb +50 -0
- data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +1 -3
- data/lib/html2rss/selectors/post_processors/parse_time.rb +5 -6
- data/lib/html2rss/selectors/post_processors/parse_uri.rb +1 -1
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +18 -5
- data/lib/html2rss/selectors/post_processors/substring.rb +1 -1
- data/lib/html2rss/selectors/schema_export.rb +60 -60
- data/lib/html2rss/selectors.rb +67 -30
- data/lib/html2rss/sst/attrs.rb +2 -2
- data/lib/html2rss/sst/document.rb +3 -3
- data/lib/html2rss/sst/index.rb +30 -11
- data/lib/html2rss/sst/node.rb +42 -29
- data/lib/html2rss/sst/normalizer.rb +47 -34
- data/lib/html2rss/sst/tags.rb +113 -2
- data/lib/html2rss/sst/text.rb +94 -58
- data/lib/html2rss/sst.rb +2 -0
- data/lib/html2rss/status.rb +27 -31
- data/lib/html2rss/surface_category.rb +1 -1
- data/lib/html2rss/syndication/README.md +12 -6
- data/lib/html2rss/syndication/alternate_links.rb +39 -0
- data/lib/html2rss/syndication/channel_facts.rb +87 -0
- data/lib/html2rss/syndication/discovery.rb +171 -187
- data/lib/html2rss/syndication/feed.rb +20 -0
- data/lib/html2rss/syndication/item_media.rb +103 -0
- data/lib/html2rss/syndication/parser.rb +117 -116
- data/lib/html2rss/test/enhance_audit.rb +15 -17
- data/lib/html2rss/test/item_sample.rb +43 -0
- data/lib/html2rss/test/policy.rb +9 -11
- data/lib/html2rss/test.rb +66 -57
- data/lib/html2rss/url.rb +163 -62
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss/word_count.rb +53 -0
- data/lib/html2rss.rb +19 -9
- data/schema/html2rss-config.schema.json +56 -41
- metadata +33 -4
- data/lib/html2rss/html/rendering/media_renderer.rb +0 -39
- data/lib/html2rss/html/rendering/pdf_renderer.rb +0 -30
|
@@ -11,6 +11,9 @@ module Html2rss
|
|
|
11
11
|
# introspection SoT (+for+ / +expectation_for+); {SchemaExport} maps Ruby
|
|
12
12
|
# types to JSON Schema only.
|
|
13
13
|
class OptionSpec
|
|
14
|
+
# Raised when an extractor/post-processor +OPTIONS+ declares a type with no JSON Schema mapping.
|
|
15
|
+
class InvalidError < Html2rss::ContractError; end
|
|
16
|
+
|
|
14
17
|
# Maps Ruby option types to JSON Schema +type+ strings.
|
|
15
18
|
RUBY_TO_JSON_TYPE = {
|
|
16
19
|
String => 'string',
|
|
@@ -83,7 +86,7 @@ module Html2rss
|
|
|
83
86
|
when Array then ruby_type.map { |t| json_type_for(t) }
|
|
84
87
|
else
|
|
85
88
|
RUBY_TO_JSON_TYPE.fetch(ruby_type) do
|
|
86
|
-
raise
|
|
89
|
+
raise InvalidError, "unsupported OPTIONS type mapping for #{ruby_type}"
|
|
87
90
|
end
|
|
88
91
|
end
|
|
89
92
|
end
|
|
@@ -114,14 +114,6 @@ module Html2rss
|
|
|
114
114
|
end
|
|
115
115
|
|
|
116
116
|
attr_reader :value, :context
|
|
117
|
-
|
|
118
|
-
# Abstract method to be implemented by subclasses
|
|
119
|
-
#
|
|
120
|
-
# @return [Object] transformed value
|
|
121
|
-
# @raise [NotImplementedError] if not implemented in subclass
|
|
122
|
-
def call
|
|
123
|
-
raise NotImplementedError, 'You must implement the `call` method in the post processor'
|
|
124
|
-
end
|
|
125
117
|
end
|
|
126
118
|
end
|
|
127
119
|
end
|
|
@@ -70,7 +70,7 @@ module Html2rss
|
|
|
70
70
|
#
|
|
71
71
|
# @param string [String]
|
|
72
72
|
# @return [Regexp]
|
|
73
|
-
# @raise [
|
|
73
|
+
# @raise [InvalidOption] when +string+ is not a String, exceeds the character cap,
|
|
74
74
|
# contains nested quantifiers, or does not parse
|
|
75
75
|
def self.compiled_pattern(string)
|
|
76
76
|
compiled_patterns[string] ||= compile_regexp_string(string)
|
|
@@ -88,25 +88,25 @@ module Html2rss
|
|
|
88
88
|
# to maintain backwards compatibility before building the Regexp.
|
|
89
89
|
# Length is the character count of the authored string (the JSON Schema
|
|
90
90
|
# +maxLength+ unit), then the AST is walked for nested quantifiers,
|
|
91
|
-
# then +to_re+. Parser errors are re-raised as
|
|
91
|
+
# then +to_re+. Parser errors are re-raised as InvalidOption so
|
|
92
92
|
# validation and execution share one path.
|
|
93
93
|
#
|
|
94
94
|
# @param string [String]
|
|
95
95
|
# @return [Regexp]
|
|
96
|
-
# @raise [
|
|
96
|
+
# @raise [InvalidOption] when +string+ is not a String, exceeds {MAX_PATTERN_LENGTH},
|
|
97
97
|
# contains nested quantifiers, or does not parse
|
|
98
98
|
def self.compile_regexp_string(string)
|
|
99
|
-
raise
|
|
100
|
-
raise
|
|
99
|
+
raise InvalidOption, 'must be a string!' unless string.is_a?(String)
|
|
100
|
+
raise InvalidOption, "pattern exceeds #{MAX_PATTERN_LENGTH} characters" if string.length > MAX_PATTERN_LENGTH
|
|
101
101
|
|
|
102
102
|
source = regexp_source(string)
|
|
103
103
|
|
|
104
|
-
expression = Regexp::Parser.parse(source
|
|
105
|
-
raise
|
|
104
|
+
expression = Regexp::Parser.parse(source)
|
|
105
|
+
raise InvalidOption, 'pattern contains nested quantifiers' if nested_quantifiers?(expression)
|
|
106
106
|
|
|
107
107
|
expression.to_re
|
|
108
108
|
rescue Regexp::Parser::Error => error
|
|
109
|
-
raise
|
|
109
|
+
raise InvalidOption, error.message, [], cause: nil
|
|
110
110
|
end
|
|
111
111
|
private_class_method :compile_regexp_string
|
|
112
112
|
|
|
@@ -161,8 +161,7 @@ module Html2rss
|
|
|
161
161
|
##
|
|
162
162
|
# @return [Regexp]
|
|
163
163
|
def pattern
|
|
164
|
-
|
|
165
|
-
raw.is_a?(String) ? self.class.compiled_pattern(raw) : raw
|
|
164
|
+
self.class.compiled_pattern(@pattern)
|
|
166
165
|
end
|
|
167
166
|
end
|
|
168
167
|
end
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Html2rss
|
|
4
|
+
class Selectors
|
|
5
|
+
module PostProcessors
|
|
6
|
+
module HtmlTransformers
|
|
7
|
+
##
|
|
8
|
+
# Transformer that keeps only media a reader can play: an +http(s)+ +src+ on +<audio>+, +<video>+
|
|
9
|
+
# and +<source>+, made absolute; any other +src+ (missing, blank, +javascript:+, +data:+, unparsable)
|
|
10
|
+
# is removed. A +<source>+ left without one is dropped, and so is an +<audio>+ or +<video>+ with no
|
|
11
|
+
# playable +src+ of its own or on a child +<source>+ (no empty player shell).
|
|
12
|
+
class KeepPlayableMedia
|
|
13
|
+
# Elements that play a file named by +src+ (theirs or a child +<source>+).
|
|
14
|
+
PLAYERS = %w[audio video].freeze
|
|
15
|
+
|
|
16
|
+
# @param base_url [String, Html2rss::Url] base URL used to resolve relative sources
|
|
17
|
+
def initialize(base_url)
|
|
18
|
+
@base_url = base_url
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
##
|
|
22
|
+
# @param node_name [String] node name currently being transformed
|
|
23
|
+
# @param node [Nokogiri::XML::Node] node currently being transformed
|
|
24
|
+
# @param _env [Hash] transformer context
|
|
25
|
+
# @option _env [Object] :_reserved reserved for transformer pipeline context
|
|
26
|
+
# @return [void]
|
|
27
|
+
def call(node_name:, node:, **_env)
|
|
28
|
+
if node_name == 'source'
|
|
29
|
+
node.unlink unless absolutize_src(node)
|
|
30
|
+
elsif PLAYERS.include?(node_name)
|
|
31
|
+
# Children are transformed after their parent, so child sources are only checked here.
|
|
32
|
+
playable = absolutize_src(node) || node.css('source').any? { Url.resolve_web(_1['src'], @base_url) }
|
|
33
|
+
node.unlink unless playable
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
# Rewrites +src+ to its absolute playable URL (Nokogiri stores its +to_s+) or removes it;
|
|
40
|
+
# returns the URL kept, if any.
|
|
41
|
+
def absolutize_src(node)
|
|
42
|
+
url = Url.resolve_web(node['src'], @base_url)
|
|
43
|
+
url ? node['src'] = url : node.remove_attribute('src')
|
|
44
|
+
url
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb
CHANGED
|
@@ -12,9 +12,7 @@ module Html2rss
|
|
|
12
12
|
URL_ELEMENTS_WITH_URL_ATTRIBUTE = {
|
|
13
13
|
'a' => :href, # Visible link
|
|
14
14
|
'img' => :src, # Visible image
|
|
15
|
-
'iframe' => :src
|
|
16
|
-
'audio' => :src, # Can show controls, so potentially visible
|
|
17
|
-
'video' => :src # Video player is visible
|
|
15
|
+
'iframe' => :src # Embedded frame (visible content)
|
|
18
16
|
}.freeze
|
|
19
17
|
|
|
20
18
|
# @param base_url [String, Html2rss::Url] base URL used to resolve relative links
|
|
@@ -63,15 +63,14 @@ module Html2rss
|
|
|
63
63
|
# @param _value [String] extracted time string (unused; type-checked via VALUE_TYPE)
|
|
64
64
|
# @param context [Selectors::StepEnv] must carry a usable +time_zone+
|
|
65
65
|
# @return [void]
|
|
66
|
-
# @raise [
|
|
66
|
+
# @raise [MissingOption] when time_zone is nil or empty
|
|
67
67
|
def self.validate_args!(_value, context)
|
|
68
68
|
time_zone_value = context.time_zone
|
|
69
69
|
|
|
70
|
-
|
|
71
|
-
raise ArgumentError, 'time_zone cannot be nil or empty', [], cause: nil
|
|
72
|
-
end
|
|
70
|
+
raise MissingOption, 'time_zone cannot be nil or empty', [], cause: nil if time_zone_value.nil?
|
|
73
71
|
|
|
74
72
|
assert_type(time_zone_value, String, :time_zone, context:)
|
|
73
|
+
raise MissingOption, 'time_zone cannot be nil or empty', [], cause: nil if time_zone_value.empty?
|
|
75
74
|
end
|
|
76
75
|
|
|
77
76
|
##
|
|
@@ -89,7 +88,7 @@ module Html2rss
|
|
|
89
88
|
private
|
|
90
89
|
|
|
91
90
|
def parse_time(string, time_zone)
|
|
92
|
-
components = Date._parse(string
|
|
91
|
+
components = Date._parse(string)
|
|
93
92
|
if components[:zone] || components.key?(:offset)
|
|
94
93
|
Time.parse(string)
|
|
95
94
|
else
|
|
@@ -104,7 +103,7 @@ module Html2rss
|
|
|
104
103
|
components[:mday],
|
|
105
104
|
components[:hour] || 0,
|
|
106
105
|
components[:min] || 0,
|
|
107
|
-
|
|
106
|
+
components[:sec] || 0
|
|
108
107
|
)
|
|
109
108
|
end
|
|
110
109
|
end
|
|
@@ -44,7 +44,7 @@ module Html2rss
|
|
|
44
44
|
# @param _context [Selectors::StepEnv] post-processor context
|
|
45
45
|
# @return [void]
|
|
46
46
|
def self.validate_args!(value, _context)
|
|
47
|
-
raise
|
|
47
|
+
raise MissingOption, 'The `value` option is missing or empty.' if value.to_s.empty?
|
|
48
48
|
end
|
|
49
49
|
|
|
50
50
|
##
|
|
@@ -17,12 +17,14 @@ module Html2rss
|
|
|
17
17
|
# - `referrer-policy='no-referrer'` to <img> tags
|
|
18
18
|
# - wraps all <img> tags, whose direct parent is not an <a>, into an <a>
|
|
19
19
|
# linking to the <img>'s `src`.
|
|
20
|
+
# - keeps <audio>, <video> and <source> with an absolute http(s) `src`; a player
|
|
21
|
+
# without a playable `src` (its own or a <source>'s) is removed.
|
|
20
22
|
#
|
|
21
23
|
# Imagine this HTML structure:
|
|
22
24
|
#
|
|
23
25
|
# <section>
|
|
24
26
|
# Lorem <b>ipsum</b> dolor...
|
|
25
|
-
# <iframe src="https://evil.
|
|
27
|
+
# <iframe src="https://evil.example/miner"></iframe>
|
|
26
28
|
# <script>alert();</script>
|
|
27
29
|
# </section>
|
|
28
30
|
#
|
|
@@ -93,10 +95,20 @@ module Html2rss
|
|
|
93
95
|
'audio' => {
|
|
94
96
|
'referrerpolicy' => 'no-referrer',
|
|
95
97
|
'crossorigin' => 'anonymous',
|
|
96
|
-
'preload' => 'none'
|
|
98
|
+
'preload' => 'none',
|
|
99
|
+
'controls' => 'true'
|
|
97
100
|
}
|
|
98
101
|
}.freeze
|
|
99
102
|
|
|
103
|
+
# Media elements kept in descriptions and the attributes they keep besides the html2rss defaults
|
|
104
|
+
# ({TAG_ATTRIBUTES}); {HtmlTransformers::KeepPlayableMedia} keeps +src+ only as an absolute
|
|
105
|
+
# +http(s)+ URL and drops a player left without one.
|
|
106
|
+
MEDIA_ATTRIBUTES = {
|
|
107
|
+
'audio' => %w[src],
|
|
108
|
+
'video' => %w[src],
|
|
109
|
+
'source' => %w[src type]
|
|
110
|
+
}.freeze
|
|
111
|
+
|
|
100
112
|
# @return [Hash{Symbol => Object}] JSON Schema fragment for this post-processor
|
|
101
113
|
def self.schema_export = SchemaExport.for_post_processor(name: :sanitize_html, klass: self)
|
|
102
114
|
|
|
@@ -145,24 +157,25 @@ module Html2rss
|
|
|
145
157
|
##
|
|
146
158
|
# @param base_url [String, Html2rss::Url]
|
|
147
159
|
# @return [Hash] the memoized sanitize configuration
|
|
148
|
-
# rubocop:disable-next Metrics/MethodLength, ThreadSafety/ClassInstanceVariable
|
|
160
|
+
# rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize, ThreadSafety/ClassInstanceVariable
|
|
149
161
|
def self.sanitize_config(base_url)
|
|
150
162
|
@sanitize_configs ||= {}
|
|
151
163
|
@sanitize_configs[base_url] ||= begin
|
|
152
164
|
config = Sanitize::Config.merge(
|
|
153
165
|
Sanitize::Config::RELAXED,
|
|
154
|
-
attributes: { all: %w[dir lang alt title translate] },
|
|
166
|
+
attributes: { all: %w[dir lang alt title translate], **MEDIA_ATTRIBUTES },
|
|
155
167
|
add_attributes: TAG_ATTRIBUTES,
|
|
156
168
|
transformers: [
|
|
157
169
|
lambda { |env|
|
|
158
170
|
HtmlTransformers::TransformUrlsToAbsoluteOnes.new(base_url).call(**env)
|
|
159
171
|
},
|
|
172
|
+
->(env) { HtmlTransformers::KeepPlayableMedia.new(base_url).call(**env) },
|
|
160
173
|
->(env) { HtmlTransformers::WrapImgInA.new.call(**env) }
|
|
161
174
|
]
|
|
162
175
|
)
|
|
163
176
|
config[:elements].delete('style')
|
|
164
177
|
config.delete(:css)
|
|
165
|
-
config[:elements].push(
|
|
178
|
+
config[:elements].push(*MEDIA_ATTRIBUTES.keys)
|
|
166
179
|
config.freeze
|
|
167
180
|
end
|
|
168
181
|
end
|
|
@@ -70,7 +70,7 @@ module Html2rss
|
|
|
70
70
|
return (start..) unless step_config.key?(:end)
|
|
71
71
|
|
|
72
72
|
finish = step_config[:end]
|
|
73
|
-
raise
|
|
73
|
+
raise InvalidOption, 'The `start` value must be unequal to the `end` value.' if start == finish
|
|
74
74
|
|
|
75
75
|
(start..finish)
|
|
76
76
|
end
|
|
@@ -8,72 +8,72 @@ module Html2rss
|
|
|
8
8
|
# Each registry class owns +DESCRIPTION+, +EXAMPLES+, and optional +OPTIONS+;
|
|
9
9
|
# this module is the export adapter only.
|
|
10
10
|
module SchemaExport
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
end
|
|
11
|
+
class << self
|
|
12
|
+
##
|
|
13
|
+
# @param name [Symbol, String] registry key
|
|
14
|
+
# @param klass [Class] post-processor class
|
|
15
|
+
# @return [Hash{Symbol => Object}] JSON Schema object for one post-processor
|
|
16
|
+
def for_post_processor(name:, klass:)
|
|
17
|
+
name = name.to_s
|
|
18
|
+
{
|
|
19
|
+
type: 'object',
|
|
20
|
+
title: name,
|
|
21
|
+
description: klass::DESCRIPTION,
|
|
22
|
+
examples: klass::EXAMPLES,
|
|
23
|
+
properties: post_processor_properties(name, klass),
|
|
24
|
+
required: post_processor_required(klass),
|
|
25
|
+
additionalProperties: true
|
|
26
|
+
}
|
|
27
|
+
end
|
|
29
28
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
29
|
+
##
|
|
30
|
+
# @param name [Symbol, String] registry key
|
|
31
|
+
# @param klass [Class] extractor class
|
|
32
|
+
# @return [Hash{Symbol => Object}] JSON Schema for one extractor name
|
|
33
|
+
def for_extractor(name:, klass:)
|
|
34
|
+
name = name.to_s
|
|
35
|
+
{
|
|
36
|
+
type: 'string',
|
|
37
|
+
const: name,
|
|
38
|
+
title: name,
|
|
39
|
+
description: klass::DESCRIPTION,
|
|
40
|
+
examples: klass::EXAMPLES
|
|
41
|
+
}
|
|
42
|
+
end
|
|
44
43
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
44
|
+
##
|
|
45
|
+
# @param name [String] registry key
|
|
46
|
+
# @param klass [Class]
|
|
47
|
+
# @return [Hash{Symbol => Hash}]
|
|
48
|
+
def post_processor_properties(name, klass)
|
|
49
|
+
properties = { name: { type: 'string', const: name } }
|
|
50
|
+
OptionSpec.for(klass).each do |spec|
|
|
51
|
+
properties[spec.name] = property_schema_for(spec.type)
|
|
52
|
+
end
|
|
53
|
+
properties
|
|
53
54
|
end
|
|
54
|
-
properties
|
|
55
|
-
end
|
|
56
55
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
56
|
+
##
|
|
57
|
+
# @param klass [Class]
|
|
58
|
+
# @return [Array<String>]
|
|
59
|
+
def post_processor_required(klass)
|
|
60
|
+
required = ['name']
|
|
61
|
+
required + OptionSpec.for(klass).select(&:required).map { |spec| spec.name.to_s }
|
|
62
|
+
end
|
|
64
63
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
64
|
+
##
|
|
65
|
+
# @param ruby_type [Class, Array<Class>]
|
|
66
|
+
# @return [Hash{Symbol => String, Array<String>}]
|
|
67
|
+
def property_schema_for(ruby_type)
|
|
68
|
+
{ type: json_type_for(ruby_type) }
|
|
69
|
+
end
|
|
71
70
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
71
|
+
##
|
|
72
|
+
# @param ruby_type [Class, Array<Class>]
|
|
73
|
+
# @return [String, Array<String>]
|
|
74
|
+
def json_type_for(ruby_type)
|
|
75
|
+
OptionSpec.json_type_for(ruby_type)
|
|
76
|
+
end
|
|
77
77
|
end
|
|
78
78
|
end
|
|
79
79
|
end
|
data/lib/html2rss/selectors.rb
CHANGED
|
@@ -35,6 +35,10 @@ module Html2rss
|
|
|
35
35
|
# Config selector keys that map onto a different {Article} attribute.
|
|
36
36
|
# +:enclosure+ stays singular in YAML; Article stores +:enclosures+.
|
|
37
37
|
SELECTOR_TO_ARTICLE_KEY = { enclosure: :enclosures }.freeze
|
|
38
|
+
# Admission-drop reason tallied for selector items that carry no URL (same shape as
|
|
39
|
+
# {AutoSource#admission_drops}).
|
|
40
|
+
MISSING_URL_REASON = 'missing_url'
|
|
41
|
+
|
|
38
42
|
# Selector keys that may be copied onto an Article (PROVIDED_KEYS + mapped aliases).
|
|
39
43
|
SELECTABLE_SELECTOR_KEYS = Set[*(Html2rss::Article::PROVIDED_KEYS + SELECTOR_TO_ARTICLE_KEY.keys)].freeze
|
|
40
44
|
# ArticleExtractor keys that enhance can fill. Skip the extractor when all are already present.
|
|
@@ -56,6 +60,7 @@ module Html2rss
|
|
|
56
60
|
@time_zone = time_zone
|
|
57
61
|
@rss_item_attributes = @selectors.keys.select { SELECTABLE_SELECTOR_KEYS.include?(_1) }
|
|
58
62
|
@parsed_body = nil
|
|
63
|
+
@unlinked_count = 0
|
|
59
64
|
@compiled_fields = compile_selector_fields
|
|
60
65
|
@compiled_item_fields = @rss_item_attributes.filter_map { @compiled_fields[_1] }
|
|
61
66
|
end
|
|
@@ -70,39 +75,52 @@ module Html2rss
|
|
|
70
75
|
end
|
|
71
76
|
|
|
72
77
|
##
|
|
73
|
-
#
|
|
78
|
+
# Drop tallies for the last iteration: items skipped for having no URL, as {AutoSource#admission_drops}.
|
|
79
|
+
#
|
|
80
|
+
# @return [Hash{String => Integer}] empty when every item was linked
|
|
81
|
+
def admission_drops
|
|
82
|
+
@unlinked_count.positive? ? { MISSING_URL_REASON => @unlinked_count }.freeze : {}.freeze
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
##
|
|
86
|
+
# Iterates over each scraped article. An item without a URL is skipped and tallied in {#admission_drops},
|
|
87
|
+
# as in auto-source ({Html2rss::Article#linked?}).
|
|
74
88
|
#
|
|
75
89
|
# @yield [article] Gives each article as an Html2rss::Article.
|
|
76
90
|
# @return [Enumerator] An enumerator if no block is given.
|
|
77
91
|
def each(&)
|
|
78
|
-
return enum_for
|
|
92
|
+
return enum_for unless block_given?
|
|
79
93
|
|
|
80
94
|
enhance = enhance?
|
|
95
|
+
@unlinked_count = 0
|
|
81
96
|
|
|
82
97
|
parsed_body.css(items_selector).each do |item|
|
|
83
98
|
article_hash = extract_article(item)
|
|
84
99
|
|
|
85
100
|
enhance_article_hash(article_hash, item) if enhance
|
|
86
101
|
|
|
87
|
-
|
|
102
|
+
article = build_article(article_hash)
|
|
103
|
+
article.linked? ? yield(article) : @unlinked_count += 1
|
|
88
104
|
end
|
|
89
105
|
end
|
|
90
106
|
|
|
91
107
|
##
|
|
92
108
|
# Yields +baseline+, +enhanced+, and the item node per list entry for enhance diagnostics.
|
|
109
|
+
#
|
|
110
|
+
# @param linked_only [Boolean] skip items {#each} would drop for having no URL
|
|
93
111
|
# Consumed by {Test::EnhanceAudit} so the extraction loop has one owner.
|
|
94
112
|
#
|
|
95
113
|
# @yieldparam baseline [Hash]
|
|
96
114
|
# @yieldparam enhanced [Hash]
|
|
97
115
|
# @yieldparam item [Nokogiri::XML::Element]
|
|
98
116
|
# @return [Enumerator] if no block is given
|
|
99
|
-
def each_enhance_pair
|
|
100
|
-
return enum_for(:each_enhance_pair) unless block_given?
|
|
117
|
+
def each_enhance_pair(linked_only: false)
|
|
118
|
+
return enum_for(:each_enhance_pair, linked_only:) unless block_given?
|
|
101
119
|
|
|
102
120
|
parsed_body.css(items_selector).each do |item|
|
|
103
121
|
baseline = extract_article(item)
|
|
104
122
|
enhanced = enhance_article_hash(baseline.dup, item)
|
|
105
|
-
yield baseline, enhanced, item
|
|
123
|
+
yield baseline, enhanced, item unless linked_only && !build_article(enhanced).linked?
|
|
106
124
|
end
|
|
107
125
|
end
|
|
108
126
|
|
|
@@ -132,7 +150,7 @@ module Html2rss
|
|
|
132
150
|
|
|
133
151
|
field = @compiled_fields[name]
|
|
134
152
|
return select_compiled(field, scope) if field
|
|
135
|
-
return select_field(name, scope
|
|
153
|
+
return select_field(name, scope:) if name == :url
|
|
136
154
|
|
|
137
155
|
raise InvalidSelectorName, "Selector for '#{name}' is not defined."
|
|
138
156
|
end
|
|
@@ -168,6 +186,12 @@ module Html2rss
|
|
|
168
186
|
text.empty? || text == DEFAULT_ITEMS_SELECTOR
|
|
169
187
|
end
|
|
170
188
|
|
|
189
|
+
##
|
|
190
|
+
# Whether +selectors.items.enhance+ is on: each matched item node fills fields that explicit
|
|
191
|
+
# selectors left empty (see {ENHANCEABLE_KEYS}) via {Html::ArticleExtractor}, using only the
|
|
192
|
+
# item's own markup and making no extra requests.
|
|
193
|
+
#
|
|
194
|
+
# @return [Boolean]
|
|
171
195
|
def enhance? = !!@selectors.dig(ITEMS_SELECTOR_KEY, :enhance)
|
|
172
196
|
|
|
173
197
|
def parsed_body
|
|
@@ -183,6 +207,8 @@ module Html2rss
|
|
|
183
207
|
end
|
|
184
208
|
end
|
|
185
209
|
|
|
210
|
+
def build_article(article_hash) = Article.new(**article_hash, scraper: self.class)
|
|
211
|
+
|
|
186
212
|
def extract_article(item)
|
|
187
213
|
scope = item_env_for(item, @url)
|
|
188
214
|
hash = @compiled_item_fields.each_with_object({}) do |field, h|
|
|
@@ -190,13 +216,30 @@ module Html2rss
|
|
|
190
216
|
next if value.nil?
|
|
191
217
|
|
|
192
218
|
article_key = field.article_key
|
|
193
|
-
h[article_key] = article_key
|
|
219
|
+
h[article_key] = article_value(article_key, value, scope)
|
|
194
220
|
end
|
|
195
221
|
|
|
196
222
|
hash[:url] ||= default_item_url(item) if anchor_element?(item)
|
|
197
223
|
hash
|
|
198
224
|
end
|
|
199
225
|
|
|
226
|
+
def article_value(article_key, value, scope)
|
|
227
|
+
case article_key
|
|
228
|
+
when :enclosures then wrap_enclosure_value(value)
|
|
229
|
+
when :url then resolve_url_value(value, scope.base_url)
|
|
230
|
+
else value
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# A +url+ selector value that is a single whitespace-free string token (a +static+ value or a post-processor
|
|
235
|
+
# result) resolves against the page URL like an +href+; a value that is no http(s) reference stays as given.
|
|
236
|
+
# Text with surrounding words is left to +Article#url+'s sanitizer, which extracts the embedded URL.
|
|
237
|
+
def resolve_url_value(value, base_url)
|
|
238
|
+
return value unless value.is_a?(String) && value.match?(/\A\S+\z/)
|
|
239
|
+
|
|
240
|
+
Url.resolve_web(value, base_url) || value
|
|
241
|
+
end
|
|
242
|
+
|
|
200
243
|
def enhance_article_hash(article_hash, article_tag) # rubocop:disable Metrics/MethodLength -- merge extracted keys
|
|
201
244
|
return article_hash if enhanceable_fields_present?(article_hash)
|
|
202
245
|
|
|
@@ -205,7 +248,6 @@ module Html2rss
|
|
|
205
248
|
article_tag,
|
|
206
249
|
base_url: @url,
|
|
207
250
|
selected_anchor:,
|
|
208
|
-
fallback_anchorless: true,
|
|
209
251
|
time_zone: @time_zone
|
|
210
252
|
)
|
|
211
253
|
return article_hash unless extracted
|
|
@@ -225,15 +267,11 @@ module Html2rss
|
|
|
225
267
|
value.is_a?(Array) ? value : [value]
|
|
226
268
|
end
|
|
227
269
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
270
|
+
# Only reached for +:url+ without a configured selector, so the item must be an anchor.
|
|
271
|
+
def select_field(selector_key, scope:)
|
|
272
|
+
return default_item_url(scope.item, scope.base_url) if anchor_element?(scope.item)
|
|
231
273
|
|
|
232
|
-
|
|
233
|
-
select_special(selector_key, scope:, config:)
|
|
234
|
-
else
|
|
235
|
-
select_regular(selector_key, scope:, config:)
|
|
236
|
-
end
|
|
274
|
+
raise InvalidSelectorName, "Selector for '#{selector_key}' is not defined."
|
|
237
275
|
end
|
|
238
276
|
|
|
239
277
|
def select_compiled(field, scope)
|
|
@@ -327,6 +365,7 @@ module Html2rss
|
|
|
327
365
|
end
|
|
328
366
|
|
|
329
367
|
def post_process(scope, value, post_process_steps)
|
|
368
|
+
value = step_input(value)
|
|
330
369
|
post_process_steps.each do |step_config|
|
|
331
370
|
value = PostProcessors.call(step_config[:name], value, scope.context_for(step_config:))
|
|
332
371
|
end
|
|
@@ -334,9 +373,15 @@ module Html2rss
|
|
|
334
373
|
value
|
|
335
374
|
end
|
|
336
375
|
|
|
376
|
+
# Post-processors work on text, so a URL value (from the href extractor) enters them as a String.
|
|
377
|
+
def step_input(value)
|
|
378
|
+
value.is_a?(Url) ? value.to_s : value
|
|
379
|
+
end
|
|
380
|
+
|
|
337
381
|
def apply_compiled_post_process(scope, value, steps)
|
|
338
382
|
return value unless value && steps
|
|
339
383
|
|
|
384
|
+
value = step_input(value)
|
|
340
385
|
steps.each do |klass, step_config|
|
|
341
386
|
value = klass.new(value, scope.context_for(step_config:)).call
|
|
342
387
|
end
|
|
@@ -375,15 +420,8 @@ module Html2rss
|
|
|
375
420
|
end
|
|
376
421
|
|
|
377
422
|
def extract_category_text(category)
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
Html2rss::Html::Navigator.extract_visible_text(category)
|
|
381
|
-
else
|
|
382
|
-
category&.to_s
|
|
383
|
-
end
|
|
384
|
-
|
|
385
|
-
stripped = text&.strip
|
|
386
|
-
stripped unless stripped.nil? || stripped.empty?
|
|
423
|
+
stripped = category.to_s.strip
|
|
424
|
+
stripped unless stripped.empty?
|
|
387
425
|
end
|
|
388
426
|
|
|
389
427
|
def node_set_with_multiple_elements?(nodes)
|
|
@@ -396,12 +434,11 @@ module Html2rss
|
|
|
396
434
|
Extractors.element(item, config[:selector])
|
|
397
435
|
end
|
|
398
436
|
|
|
399
|
-
# @return [Hash, nil] enclosure details, or nil when the selector yields nothing
|
|
437
|
+
# @return [Hash, nil] enclosure details, or nil when the selector yields nothing or no http(s) URL
|
|
438
|
+
# ({Url.resolve_web})
|
|
400
439
|
def enclosure(scope:, config:)
|
|
401
440
|
selected = select_regular(:enclosure, scope:, config:)
|
|
402
|
-
return
|
|
403
|
-
|
|
404
|
-
url = Url.from_relative(selected, scope.base_url)
|
|
441
|
+
return unless (url = Url.resolve_web(selected, scope.base_url))
|
|
405
442
|
|
|
406
443
|
{ url:, type: config[:content_type] }
|
|
407
444
|
end
|
data/lib/html2rss/sst/attrs.rb
CHANGED
|
@@ -65,7 +65,7 @@ module Html2rss
|
|
|
65
65
|
case class_names
|
|
66
66
|
when nil then AttrDefaults::EMPTY_CLASS_NAMES
|
|
67
67
|
when Array then class_names.map { |token| token.to_s.freeze }.freeze
|
|
68
|
-
else raise
|
|
68
|
+
else raise InvalidError, "class_names must be an Array, got #{class_names.class}"
|
|
69
69
|
end
|
|
70
70
|
end
|
|
71
71
|
|
|
@@ -73,7 +73,7 @@ module Html2rss
|
|
|
73
73
|
case raw
|
|
74
74
|
when nil then AttrDefaults::EMPTY_RAW
|
|
75
75
|
when Hash then raw.transform_values { |v| v.to_s.freeze }.freeze
|
|
76
|
-
else raise
|
|
76
|
+
else raise InvalidError, "raw must be a Hash, got #{raw.class}"
|
|
77
77
|
end
|
|
78
78
|
end
|
|
79
79
|
end
|
|
@@ -13,10 +13,10 @@ module Html2rss
|
|
|
13
13
|
# @param degraded [Boolean] true when MAX_NODES forced semantic-tag-only mode
|
|
14
14
|
# @param node_count [Integer]
|
|
15
15
|
# @return [Document]
|
|
16
|
-
# @raise [
|
|
16
|
+
# @raise [InvalidError] when root/index missing
|
|
17
17
|
def self.build(root:, index:, degraded: false, node_count: 0)
|
|
18
|
-
raise
|
|
19
|
-
raise
|
|
18
|
+
raise InvalidError, 'root is required' unless root.is_a?(Node)
|
|
19
|
+
raise InvalidError, 'index is required' unless index.is_a?(Index)
|
|
20
20
|
|
|
21
21
|
new(root:, index:, degraded: !!degraded, node_count: Integer(node_count))
|
|
22
22
|
end
|