html2rss 0.32.4 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -2
- data/lib/html2rss/article/deduplicator.rb +5 -2
- data/lib/html2rss/article/enclosure.rb +185 -25
- data/lib/html2rss/article/title.rb +203 -0
- data/lib/html2rss/article.rb +79 -13
- data/lib/html2rss/auto_source/README.md +16 -8
- data/lib/html2rss/auto_source/cleanup.rb +127 -53
- data/lib/html2rss/auto_source/corroboration.rb +167 -0
- data/lib/html2rss/auto_source/page_links/anchor_claims.rb +82 -0
- data/lib/html2rss/auto_source/page_links/card_units.rb +77 -0
- data/lib/html2rss/auto_source/page_links/label_binding.rb +79 -0
- data/lib/html2rss/auto_source/page_links/sponsored_cards.rb +147 -0
- data/lib/html2rss/auto_source/page_links.rb +164 -0
- data/lib/html2rss/auto_source/scraper/html.rb +11 -45
- data/lib/html2rss/auto_source/scraper/json_document.rb +3 -6
- data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +202 -125
- data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +27 -38
- data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +123 -123
- data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +27 -35
- data/lib/html2rss/auto_source/scraper/json_state.rb +4 -9
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +27 -5
- data/lib/html2rss/auto_source/scraper/microdata.rb +233 -248
- data/lib/html2rss/auto_source/scraper/microformats2.rb +10 -4
- data/lib/html2rss/auto_source/scraper/native_feed.rb +28 -18
- data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +7 -2
- data/lib/html2rss/auto_source/scraper/schema/item_list.rb +5 -3
- data/lib/html2rss/auto_source/scraper/schema/thing.rb +140 -22
- data/lib/html2rss/auto_source/scraper/schema.rb +52 -8
- data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb +3 -9
- data/lib/html2rss/auto_source/scraper/semantic_html.rb +12 -20
- data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +1 -4
- data/lib/html2rss/auto_source/scraper/sitemap.rb +7 -14
- data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +2 -2
- data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb +1 -3
- data/lib/html2rss/auto_source/scraper/wordpress_api.rb +7 -7
- data/lib/html2rss/auto_source/scraper/xhr_articles.rb +2 -4
- data/lib/html2rss/auto_source/scraper.rb +10 -13
- data/lib/html2rss/auto_source/segment.rb +19 -5
- data/lib/html2rss/auto_source/segmenter/cluster.rb +18 -22
- data/lib/html2rss/auto_source/segmenter/list.rb +35 -59
- data/lib/html2rss/auto_source/segmenter/primary_link.rb +29 -4
- data/lib/html2rss/auto_source/segmenter/semantic.rb +23 -19
- data/lib/html2rss/auto_source/segmenter.rb +4 -8
- data/lib/html2rss/auto_source.rb +81 -26
- data/lib/html2rss/batch.rb +24 -32
- data/lib/html2rss/capture/README.md +6 -3
- data/lib/html2rss/capture.rb +51 -59
- data/lib/html2rss/channel.rb +30 -12
- data/lib/html2rss/cli/render.rb +97 -96
- data/lib/html2rss/cli/validate.rb +2 -4
- data/lib/html2rss/cli.rb +55 -38
- data/lib/html2rss/config/auto_source_contract.rb +52 -45
- data/lib/html2rss/config/issue_mapper.rb +4 -1
- data/lib/html2rss/config/request_controls.rb +16 -5
- data/lib/html2rss/config/schema.rb +23 -15
- data/lib/html2rss/config/selectors_validator.rb +3 -2
- data/lib/html2rss/config/validation_issue.rb +4 -1
- data/lib/html2rss/config/validator.rb +14 -3
- data/lib/html2rss/config.rb +54 -28
- data/lib/html2rss/contract_error.rb +9 -0
- data/lib/html2rss/defaults.rb +25 -38
- data/lib/html2rss/doctor/botasaurus.rb +18 -22
- data/lib/html2rss/error.rb +17 -3
- data/lib/html2rss/error_kind.rb +39 -0
- data/lib/html2rss/feed_builder/item_presentation.rb +22 -9
- data/lib/html2rss/feed_builder/json_feed/item.rb +2 -1
- data/lib/html2rss/feed_builder/json_feed.rb +2 -2
- data/lib/html2rss/feed_builder/rss/stylesheet.rb +3 -3
- data/lib/html2rss/feed_builder/rss.rb +50 -7
- data/lib/html2rss/feed_builder.rb +2 -1
- data/lib/html2rss/feed_pipeline/README.md +1 -1
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +16 -34
- data/lib/html2rss/feed_pipeline/strategy_plan.rb +3 -3
- data/lib/html2rss/feed_pipeline.rb +40 -27
- data/lib/html2rss/feed_resolution/README.md +1 -1
- data/lib/html2rss/feed_resolution/candidate_generator.rb +6 -8
- data/lib/html2rss/feed_resolution/options.rb +3 -7
- data/lib/html2rss/feed_resolution/policy.rb +29 -33
- data/lib/html2rss/feed_resolution/probe.rb +6 -4
- data/lib/html2rss/feed_resolution/scorer.rb +29 -29
- data/lib/html2rss/feed_resolution.rb +16 -21
- data/lib/html2rss/feed_result.rb +11 -6
- data/lib/html2rss/hash_util.rb +91 -90
- data/lib/html2rss/html/article_extractor/category_extractor.rb +7 -10
- data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
- data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +12 -33
- data/lib/html2rss/html/article_extractor/heading_extractor.rb +2 -3
- data/lib/html2rss/html/article_extractor/id_generator.rb +7 -10
- data/lib/html2rss/html/article_extractor/image_extractor.rb +9 -1
- data/lib/html2rss/html/article_extractor.rb +19 -47
- data/lib/html2rss/html/article_rules/category.rb +3 -3
- data/lib/html2rss/html/article_rules/description.rb +6 -15
- data/lib/html2rss/html/article_rules/enclosure.rb +70 -42
- data/lib/html2rss/html/article_rules/image.rb +30 -16
- data/lib/html2rss/html/feed_link.rb +7 -9
- data/lib/html2rss/html/navigator/text_extractor.rb +26 -16
- data/lib/html2rss/html/navigator.rb +39 -9
- data/lib/html2rss/html/probe.rb +55 -51
- data/lib/html2rss/html/rendering/description_builder.rb +46 -32
- data/lib/html2rss/html/rendering.rb +2 -9
- data/lib/html2rss/html/sst_article_extractor.rb +55 -145
- data/lib/html2rss/html/title_pieces/kicker.rb +147 -0
- data/lib/html2rss/html/title_pieces/nodes.rb +107 -0
- data/lib/html2rss/html/title_pieces/nokogiri_nodes.rb +70 -0
- data/lib/html2rss/html/title_pieces/sst_nodes.rb +71 -0
- data/lib/html2rss/html/title_pieces.rb +169 -0
- data/lib/html2rss/html/visibility.rb +56 -0
- data/lib/html2rss/html.rb +2 -0
- data/lib/html2rss/link_destination/destination_facts.rb +17 -1
- data/lib/html2rss/link_destination/noise_policy.rb +11 -10
- data/lib/html2rss/link_destination/path_classifier.rb +111 -42
- data/lib/html2rss/link_destination/sponsorship/label_text.rb +82 -0
- data/lib/html2rss/link_destination/sponsorship.rb +184 -0
- data/lib/html2rss/link_destination/text_classifier.rb +104 -8
- data/lib/html2rss/mcp/README.md +13 -1
- data/lib/html2rss/mcp/config_argument.rb +14 -4
- data/lib/html2rss/mcp/contract.rb +20 -9
- data/lib/html2rss/mcp/outcome.rb +18 -17
- data/lib/html2rss/mcp/runtime.rb +53 -23
- data/lib/html2rss/mcp/server/tools.rb +9 -9
- data/lib/html2rss/mcp/server.rb +13 -4
- data/lib/html2rss/option.rb +48 -0
- data/lib/html2rss/page_recon/README.md +1 -1
- data/lib/html2rss/page_recon/diagnostics.rb +9 -9
- data/lib/html2rss/page_recon.rb +0 -4
- data/lib/html2rss/recon.rb +19 -23
- data/lib/html2rss/request_service/blocked_surface.rb +5 -8
- data/lib/html2rss/request_service/botasaurus_contract.rb +25 -3
- data/lib/html2rss/request_service/botasaurus_strategy.rb +44 -12
- data/lib/html2rss/request_service/budget.rb +3 -24
- data/lib/html2rss/request_service/compressed_body.rb +36 -48
- data/lib/html2rss/request_service/context.rb +16 -17
- data/lib/html2rss/request_service/cookie_jar.rb +89 -0
- data/lib/html2rss/request_service/cookie_store.rb +79 -0
- data/lib/html2rss/request_service/feed_advertisement.rb +72 -0
- data/lib/html2rss/request_service/httpx_strategy.rb +7 -3
- data/lib/html2rss/request_service/local_file_strategy.rb +2 -3
- data/lib/html2rss/request_service/network_guard.rb +3 -2
- data/lib/html2rss/request_service/policy.rb +55 -36
- data/lib/html2rss/request_service/replay_breaker.rb +56 -0
- data/lib/html2rss/request_service/response.rb +33 -19
- data/lib/html2rss/request_service/response_cache.rb +109 -0
- data/lib/html2rss/request_service/response_caching.rb +95 -0
- data/lib/html2rss/request_service/strategy.rb +5 -11
- data/lib/html2rss/request_service.rb +71 -12
- data/lib/html2rss/request_session/pager/base.rb +3 -10
- data/lib/html2rss/request_session/pager/custom_selector.rb +4 -4
- data/lib/html2rss/request_session/pager/offset.rb +1 -7
- data/lib/html2rss/request_session/pager/rel_next.rb +1 -1
- data/lib/html2rss/request_session/pager/url_template.rb +1 -7
- data/lib/html2rss/request_session/pager.rb +1 -1
- data/lib/html2rss/request_session.rb +6 -2
- data/lib/html2rss/scoring/cluster_scorer.rb +1 -1
- data/lib/html2rss/scoring/container_assessor.rb +12 -13
- data/lib/html2rss/scoring/engine.rb +10 -38
- data/lib/html2rss/scoring/link_resolver.rb +2 -7
- data/lib/html2rss/scoring/observation.rb +2 -2
- data/lib/html2rss/selector_candidates.rb +19 -8
- data/lib/html2rss/selectors/README.md +1 -1
- data/lib/html2rss/selectors/extractors/href.rb +2 -4
- data/lib/html2rss/selectors/json_xml.rb +33 -34
- data/lib/html2rss/selectors/option_spec.rb +4 -1
- data/lib/html2rss/selectors/post_processors/base.rb +0 -8
- data/lib/html2rss/selectors/post_processors/gsub.rb +9 -10
- data/lib/html2rss/selectors/post_processors/html_transformers/keep_playable_media.rb +50 -0
- data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +1 -3
- data/lib/html2rss/selectors/post_processors/parse_time.rb +5 -6
- data/lib/html2rss/selectors/post_processors/parse_uri.rb +1 -1
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +18 -5
- data/lib/html2rss/selectors/post_processors/substring.rb +1 -1
- data/lib/html2rss/selectors/schema_export.rb +60 -60
- data/lib/html2rss/selectors.rb +67 -30
- data/lib/html2rss/sst/attrs.rb +2 -2
- data/lib/html2rss/sst/document.rb +3 -3
- data/lib/html2rss/sst/index.rb +30 -11
- data/lib/html2rss/sst/node.rb +42 -29
- data/lib/html2rss/sst/normalizer.rb +47 -34
- data/lib/html2rss/sst/tags.rb +113 -2
- data/lib/html2rss/sst/text.rb +94 -58
- data/lib/html2rss/sst.rb +2 -0
- data/lib/html2rss/status.rb +27 -31
- data/lib/html2rss/surface_category.rb +1 -1
- data/lib/html2rss/syndication/README.md +12 -6
- data/lib/html2rss/syndication/alternate_links.rb +39 -0
- data/lib/html2rss/syndication/channel_facts.rb +87 -0
- data/lib/html2rss/syndication/discovery.rb +171 -187
- data/lib/html2rss/syndication/feed.rb +20 -0
- data/lib/html2rss/syndication/item_media.rb +103 -0
- data/lib/html2rss/syndication/parser.rb +117 -116
- data/lib/html2rss/test/enhance_audit.rb +15 -17
- data/lib/html2rss/test/item_sample.rb +43 -0
- data/lib/html2rss/test/policy.rb +9 -11
- data/lib/html2rss/test.rb +66 -57
- data/lib/html2rss/url.rb +163 -62
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss/word_count.rb +53 -0
- data/lib/html2rss.rb +19 -9
- data/schema/html2rss-config.schema.json +56 -41
- metadata +33 -4
- data/lib/html2rss/html/rendering/media_renderer.rb +0 -39
- data/lib/html2rss/html/rendering/pdf_renderer.rb +0 -30
data/lib/html2rss/html/probe.rb
CHANGED
|
@@ -15,67 +15,71 @@ module Html2rss
|
|
|
15
15
|
APPLICATION_RSS_XML = 'application/rss+xml'
|
|
16
16
|
# @see #mime_match?
|
|
17
17
|
APPLICATION_ATOM_XML = 'application/atom+xml'
|
|
18
|
+
# @see #mime_match?
|
|
19
|
+
APPLICATION_RDF_XML = 'application/rdf+xml'
|
|
20
|
+
# Every syndication format advertised by +link[rel=alternate]+ (RSS 2.0, Atom, RSS 1.0 / RDF).
|
|
21
|
+
FEED_MIME_TYPES = [APPLICATION_RSS_XML, APPLICATION_ATOM_XML, APPLICATION_RDF_XML].freeze
|
|
18
22
|
|
|
19
|
-
|
|
23
|
+
class << self
|
|
24
|
+
##
|
|
25
|
+
# @param string [String, Symbol, #to_s] wire value
|
|
26
|
+
# @return [String] folded comparison key
|
|
27
|
+
def fold(string)
|
|
28
|
+
string.to_s.downcase
|
|
29
|
+
end
|
|
20
30
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
31
|
+
##
|
|
32
|
+
# @param node [Nokogiri::XML::Node]
|
|
33
|
+
# @return [String] folded element name
|
|
34
|
+
def tag(node)
|
|
35
|
+
name = node.name
|
|
36
|
+
return fold(name) unless name.is_a?(String)
|
|
37
|
+
return name if name.ascii_only? && !name.match?(/[A-Z]/)
|
|
27
38
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
# @return [String] folded element name
|
|
31
|
-
def tag(node)
|
|
32
|
-
name = node.name
|
|
33
|
-
return fold(name) unless name.is_a?(String)
|
|
34
|
-
return name if name.ascii_only? && !name.match?(/[A-Z]/)
|
|
39
|
+
fold(name)
|
|
40
|
+
end
|
|
35
41
|
|
|
36
|
-
|
|
37
|
-
|
|
42
|
+
##
|
|
43
|
+
# @param type [String, Symbol, #to_s, nil] MIME type attribute value
|
|
44
|
+
# @return [String] media type without parameters, folded
|
|
45
|
+
def mime_base(type)
|
|
46
|
+
raw = type.to_s
|
|
47
|
+
return fold(raw) unless raw.include?(';') || raw.match?(/\A\s|\s\z/)
|
|
38
48
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
# @return [String] media type without parameters, folded
|
|
42
|
-
def mime_base(type)
|
|
43
|
-
raw = type.to_s
|
|
44
|
-
return fold(raw) unless raw.include?(';') || raw.match?(/\A\s|\s\z/)
|
|
49
|
+
fold(raw.split(';', 2).first.to_s.strip)
|
|
50
|
+
end
|
|
45
51
|
|
|
46
|
-
|
|
47
|
-
|
|
52
|
+
##
|
|
53
|
+
# @param actual [String, Symbol, #to_s, nil] observed MIME type
|
|
54
|
+
# @param expected [Array<String>] canonical MIME types to match
|
|
55
|
+
# @return [Boolean]
|
|
56
|
+
def mime_match?(actual, *expected)
|
|
57
|
+
base = mime_base(actual)
|
|
58
|
+
expected.any? { |candidate| base == candidate || base == mime_base(candidate) }
|
|
59
|
+
end
|
|
48
60
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
expected.any? { |candidate| base == candidate || base == mime_base(candidate) }
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
##
|
|
59
|
-
# @param doc [Nokogiri::XML::Node]
|
|
60
|
-
# @param mime_types [Array<String>] optional MIME filters (folded once per call)
|
|
61
|
-
# @return [Array<Nokogiri::XML::Element>]
|
|
62
|
-
def scripts(doc, *mime_types)
|
|
63
|
-
return doc.css('script') if mime_types.empty?
|
|
61
|
+
##
|
|
62
|
+
# @param doc [Nokogiri::XML::Node]
|
|
63
|
+
# @param mime_types [Array<String>] optional MIME filters (folded once per call)
|
|
64
|
+
# @return [Array<Nokogiri::XML::Element>]
|
|
65
|
+
def scripts(doc, *mime_types)
|
|
66
|
+
return doc.css('script') if mime_types.empty?
|
|
64
67
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
+
allowed = mime_types.to_set { |mime| mime_base(mime) }
|
|
69
|
+
doc.css('script').select { |script| allowed.include?(mime_base(script['type'])) }
|
|
70
|
+
end
|
|
68
71
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
72
|
+
##
|
|
73
|
+
# @param doc [Nokogiri::XML::Node]
|
|
74
|
+
# @param rel [String] link rel token
|
|
75
|
+
# @param mime [String, nil] optional MIME filter
|
|
76
|
+
# @return [Array<Nokogiri::XML::Element>]
|
|
77
|
+
def alternate_links(doc, rel:, mime: nil)
|
|
78
|
+
nodes = doc.css(%(link[rel~="#{rel}"][href]))
|
|
79
|
+
return nodes if mime.nil?
|
|
77
80
|
|
|
78
|
-
|
|
81
|
+
nodes.select { |node| mime_match?(node['type'], mime) }
|
|
82
|
+
end
|
|
79
83
|
end
|
|
80
84
|
end
|
|
81
85
|
end
|
|
@@ -3,32 +3,38 @@
|
|
|
3
3
|
module Html2rss
|
|
4
4
|
module Html
|
|
5
5
|
module Rendering
|
|
6
|
-
# Builds a sanitized article description from the base text, title, and
|
|
6
|
+
# Builds a sanitized article description from the base text, title, and at most one media element.
|
|
7
7
|
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
8
|
+
# The media element is the item's RSS enclosure when it is an audio or video file (a player), else the item
|
|
9
|
+
# image; a downloadable document (PDF, ZIP) falls back to the image. Nothing is rendered when the sanitized
|
|
10
|
+
# description already references that file: an +src+, +href+ or +srcset+ URL resolving to the same normalized
|
|
11
|
+
# absolute URL. Those attributes are parsed only when the description contains the file name (+File.basename+
|
|
12
|
+
# of its path).
|
|
10
13
|
#
|
|
11
14
|
# @example Basic usage
|
|
12
15
|
# builder = DescriptionBuilder.new(
|
|
13
16
|
# base: "Article content",
|
|
14
17
|
# title: "Article Title",
|
|
15
18
|
# url: "https://example.com",
|
|
16
|
-
#
|
|
19
|
+
# enclosure: enclosure_object,
|
|
17
20
|
# image: "https://example.com/image.jpg"
|
|
18
21
|
# )
|
|
19
22
|
# description = builder.call
|
|
20
23
|
class DescriptionBuilder
|
|
21
|
-
#
|
|
22
|
-
|
|
24
|
+
# XPath of the attributes whose URLs count as the description already showing a file.
|
|
25
|
+
URL_ATTRIBUTES = './/@src|.//@href|.//@srcset'
|
|
26
|
+
|
|
27
|
+
# Removes the first occurrence of the pattern in the text content (never inside a tag or attribute)
|
|
28
|
+
# if it starts before the range's end. Text after a stray +<+ that opens no tag is left alone.
|
|
23
29
|
#
|
|
24
30
|
# @param text [String]
|
|
25
|
-
# @param pattern [String]
|
|
31
|
+
# @param pattern [String, nil] nothing is removed without one
|
|
26
32
|
# @param end_of_range [Integer] Optional, defaults to half the text length
|
|
27
33
|
# @return [String]
|
|
28
|
-
def self.remove_pattern_from_start(text, pattern, end_of_range:
|
|
29
|
-
return text unless
|
|
34
|
+
def self.remove_pattern_from_start(text, pattern, end_of_range: text.size / 2)
|
|
35
|
+
return text unless pattern
|
|
30
36
|
|
|
31
|
-
index = text.index(pattern)
|
|
37
|
+
index = text.index(/\A(?:<[^>]*>|[^<])*?\K#{Regexp.escape(pattern)}/)
|
|
32
38
|
return text if index.nil? || index > end_of_range
|
|
33
39
|
|
|
34
40
|
"#{text[0, index]}#{text[(index + pattern.size)..]}"
|
|
@@ -36,52 +42,60 @@ module Html2rss
|
|
|
36
42
|
|
|
37
43
|
# @param base [String] The base text content for the description
|
|
38
44
|
# @param title [String] The article title (used for alt text and title removal)
|
|
39
|
-
# @param url [String, Html2rss::Url] The article URL (used for sanitization)
|
|
40
|
-
# @param
|
|
41
|
-
#
|
|
42
|
-
|
|
45
|
+
# @param url [String, Html2rss::Url] The article URL (used for sanitization and URL resolution)
|
|
46
|
+
# @param enclosure [Html2rss::Article::Enclosure, nil] the item's RSS enclosure
|
|
47
|
+
# ({FeedBuilder::ItemPresentation.rss_enclosure_for}); rendered only when audio or video
|
|
48
|
+
# @param image [String, Html2rss::Url, nil] item image URL, rendered when the enclosure is no player
|
|
49
|
+
def initialize(base:, title:, url:, enclosure:, image:)
|
|
43
50
|
@base = base.to_s
|
|
44
51
|
@title = title
|
|
45
52
|
@url = url
|
|
46
|
-
@
|
|
53
|
+
@player = enclosure if enclosure&.media?
|
|
47
54
|
@image = image
|
|
48
55
|
end
|
|
49
56
|
|
|
50
|
-
# Generates the
|
|
57
|
+
# Generates the description: the media element (unless the text already shows it) and the sanitized text.
|
|
51
58
|
#
|
|
52
59
|
# @return [String, nil] The complete description or nil if empty
|
|
53
60
|
def call
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
61
|
+
text = processed_base_description
|
|
62
|
+
file = @player&.url || @image
|
|
63
|
+
media = media_html if file && !(text && shown?(text, Url.from_absolute(file)))
|
|
57
64
|
|
|
58
|
-
|
|
59
|
-
|
|
65
|
+
parts = [media, text].compact
|
|
66
|
+
parts.join("\n\n") unless parts.empty?
|
|
60
67
|
end
|
|
61
68
|
|
|
62
69
|
private
|
|
63
70
|
|
|
64
|
-
def
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
71
|
+
def media_html
|
|
72
|
+
if @player&.video? then VideoRenderer.new(url: @player.url, type: @player.type).to_html
|
|
73
|
+
elsif @player then AudioRenderer.new(url: @player.url, type: @player.type).to_html
|
|
74
|
+
else ImageRenderer.new(url: @image, title: @title).to_html
|
|
75
|
+
end
|
|
76
|
+
end
|
|
68
77
|
|
|
69
|
-
|
|
78
|
+
def shown?(text, file)
|
|
79
|
+
text.include?(File.basename(file.path)) && referenced_urls(text).include?(file.to_s)
|
|
70
80
|
end
|
|
71
81
|
|
|
72
|
-
def
|
|
73
|
-
|
|
74
|
-
|
|
82
|
+
def referenced_urls(text)
|
|
83
|
+
Nokogiri::HTML.fragment(text).xpath(URL_ATTRIBUTES).flat_map do |attribute|
|
|
84
|
+
value = attribute.value
|
|
85
|
+
values = attribute.name == 'srcset' ? value.split(',').map { _1[/\S+/] } : [value]
|
|
86
|
+
values.map { resolve(_1) }
|
|
75
87
|
end
|
|
76
88
|
end
|
|
77
89
|
|
|
78
|
-
def
|
|
79
|
-
|
|
90
|
+
def resolve(value)
|
|
91
|
+
Url.from_relative(value, @url).to_s
|
|
92
|
+
rescue Url::InvalidUrlError
|
|
93
|
+
nil
|
|
80
94
|
end
|
|
81
95
|
|
|
82
96
|
def processed_base_description
|
|
83
97
|
text = self.class.remove_pattern_from_start(@base, @title)
|
|
84
|
-
|
|
98
|
+
Selectors::PostProcessors::SanitizeHtml.call(text, @url)
|
|
85
99
|
end
|
|
86
100
|
end
|
|
87
101
|
end
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module Html2rss
|
|
4
4
|
module Html
|
|
5
|
-
# Namespace for HTML rendering logic
|
|
6
|
-
#
|
|
5
|
+
# Namespace for HTML rendering logic: the feed item description and its one media element
|
|
6
|
+
# (an image, audio or video player).
|
|
7
7
|
#
|
|
8
8
|
# @see Html2rss::Html::Rendering::DescriptionBuilder
|
|
9
9
|
#
|
|
@@ -12,13 +12,6 @@ module Html2rss
|
|
|
12
12
|
# url: "https://example.com/image.jpg",
|
|
13
13
|
# title: "Example"
|
|
14
14
|
# ).to_html
|
|
15
|
-
#
|
|
16
|
-
# @example
|
|
17
|
-
# Html2rss::Html::Rendering::MediaRenderer.for(
|
|
18
|
-
# enclosure: nil,
|
|
19
|
-
# image: "https://example.com/image.jpg",
|
|
20
|
-
# title: "Example"
|
|
21
|
-
# )
|
|
22
15
|
module Rendering
|
|
23
16
|
end
|
|
24
17
|
end
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'zlib'
|
|
4
|
-
|
|
5
3
|
module Html2rss
|
|
6
4
|
module Html
|
|
7
5
|
##
|
|
@@ -9,10 +7,9 @@ module Html2rss
|
|
|
9
7
|
# Port of Html::ArticleExtractor field logic onto SST::Node.
|
|
10
8
|
# rubocop:disable-next Metrics/ClassLength -- SST field extractors colocated for parity with ArticleExtractor
|
|
11
9
|
class SstArticleExtractor
|
|
12
|
-
# CSS class tokens that mark
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
FALLBACK_HEADING_NAMES = Set[:strong, :b].freeze
|
|
10
|
+
# CSS class tokens that mark title-like nodes. Broad on purpose: also shared with
|
|
11
|
+
# {Html2rss::SelectorCandidates}, which owns no copy of it.
|
|
12
|
+
TITLE_CLASS_PATTERN = /title|font-bold|font-semibold/
|
|
16
13
|
# Nested blocks that mean a category node is actually a content container.
|
|
17
14
|
CATEGORY_CONTAINER_NAMES = Set[:p, :article, :section].freeze
|
|
18
15
|
|
|
@@ -21,49 +18,50 @@ module Html2rss
|
|
|
21
18
|
# @param ranked_or_segment [Scoring::RankedSegment, AutoSource::Segment]
|
|
22
19
|
# @param base_url [String, Html2rss::Url]
|
|
23
20
|
# @param scraper [Class, nil]
|
|
24
|
-
# @param fallback_anchorless [Boolean]
|
|
25
21
|
# @param time_zone [String] channel time zone for naive leftover dates
|
|
26
|
-
# @return [Html2rss::Article, nil]
|
|
27
|
-
def call(ranked_or_segment, base_url:, scraper: nil,
|
|
28
|
-
new(ranked_or_segment, base_url:, scraper:,
|
|
22
|
+
# @return [Html2rss::Article, nil] nil when the segment has no primary link (no URL of its own)
|
|
23
|
+
def call(ranked_or_segment, base_url:, scraper: nil, time_zone: 'UTC')
|
|
24
|
+
new(ranked_or_segment, base_url:, scraper:, time_zone:).call
|
|
29
25
|
end
|
|
30
26
|
end
|
|
31
27
|
|
|
32
28
|
# @param ranked_or_segment [Scoring::RankedSegment, AutoSource::Segment]
|
|
33
29
|
# @param base_url [String, Html2rss::Url]
|
|
34
30
|
# @param scraper [Class, nil]
|
|
35
|
-
# @param fallback_anchorless [Boolean]
|
|
36
31
|
# @param time_zone [String] channel time zone for naive leftover dates
|
|
37
|
-
def initialize(ranked_or_segment, base_url:, scraper: nil,
|
|
32
|
+
def initialize(ranked_or_segment, base_url:, scraper: nil, time_zone: 'UTC')
|
|
38
33
|
segment = ranked_or_segment.is_a?(Scoring::RankedSegment) ? ranked_or_segment.segment : ranked_or_segment
|
|
39
|
-
raise
|
|
34
|
+
raise InvalidError, 'segment is required' unless segment.is_a?(AutoSource::Segment)
|
|
40
35
|
|
|
41
36
|
@segment = segment
|
|
42
37
|
@root = segment.root_node
|
|
43
38
|
@selected_anchor = segment.primary_link
|
|
44
39
|
@base_url = base_url
|
|
45
40
|
@scraper = scraper
|
|
46
|
-
@fallback_anchorless = fallback_anchorless
|
|
47
41
|
@time_zone = time_zone
|
|
48
42
|
end
|
|
49
43
|
|
|
50
44
|
##
|
|
51
45
|
# @return [Html2rss::Article, nil]
|
|
52
46
|
def call # rubocop:disable Metrics/MethodLength
|
|
53
|
-
|
|
47
|
+
return unless @selected_anchor
|
|
48
|
+
|
|
49
|
+
title = title_from_in_card_sources
|
|
54
50
|
lines = leftover_lines
|
|
55
51
|
published_at = extract_published_at(lines)
|
|
56
52
|
root, lines, published_at = parent_card_fields(title, lines, published_at)
|
|
57
53
|
attrs = {
|
|
58
54
|
title:,
|
|
55
|
+
kicker: title_pieces.kicker_text,
|
|
59
56
|
url: extract_url,
|
|
60
57
|
image: extract_image,
|
|
61
58
|
description: ArticleRules::Description.from_lines(lines, title:),
|
|
62
|
-
id:
|
|
59
|
+
id: parse_id_from_dom,
|
|
63
60
|
published_at:,
|
|
64
61
|
enclosures: extract_enclosures,
|
|
65
62
|
categories: extract_categories(title, root:),
|
|
66
|
-
scraper: @scraper
|
|
63
|
+
scraper: @scraper,
|
|
64
|
+
compose_title: true
|
|
67
65
|
}
|
|
68
66
|
article = Article.new(**attrs)
|
|
69
67
|
article.valid? ? article : nil
|
|
@@ -73,33 +71,17 @@ module Html2rss
|
|
|
73
71
|
|
|
74
72
|
def extract_url
|
|
75
73
|
@extract_url ||= begin
|
|
76
|
-
href = @selected_anchor
|
|
77
|
-
|
|
78
|
-
anchorless_url_fallback
|
|
79
|
-
else
|
|
80
|
-
Url.from_relative(href.split('#').first.strip, @base_url)
|
|
81
|
-
end
|
|
74
|
+
href = @selected_anchor.attrs.href.to_s
|
|
75
|
+
Url.from_relative(href.split('#').first.strip, @base_url) unless href.empty?
|
|
82
76
|
end
|
|
83
77
|
end
|
|
84
78
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
id = generate_id
|
|
89
|
-
Url.from_relative("##{id}", @base_url) if id
|
|
90
|
-
end
|
|
91
|
-
|
|
92
|
-
def extract_title
|
|
93
|
-
title_text = title_from_in_card_sources
|
|
94
|
-
return unless title_text
|
|
79
|
+
# Kicker and headline text ({TitlePieces}) of this card.
|
|
80
|
+
def title_pieces = @title_pieces ||= TitlePieces.new(@root, heading || @selected_anchor, anchor: @selected_anchor)
|
|
95
81
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
# Prefer a non-junk title. Credit-only cards may blank to nil (description can carry
|
|
101
|
-
# the item). Other junk shapes must stay present so Cleanup can hard-reject them —
|
|
102
|
-
# blanking slug/CMS/template text would admit titleless cards.
|
|
82
|
+
# Prefer a non-junk headline ({Article} composes it with {#kicker_text}). A credit-only card
|
|
83
|
+
# blanks to nil, and Cleanup drops the untitled item as +blank_title+. Other junk shapes stay
|
|
84
|
+
# present so Cleanup rejects them under their own reason (slug, CMS, template).
|
|
103
85
|
#
|
|
104
86
|
# @return [String, nil]
|
|
105
87
|
def title_from_in_card_sources
|
|
@@ -110,14 +92,11 @@ module Html2rss
|
|
|
110
92
|
candidates.find { |text| AutoSource::Cleanup.junk_reason(text) != :credit }
|
|
111
93
|
end
|
|
112
94
|
|
|
113
|
-
# @return [Array<String>] ordered in-card title candidates (heading → anchor
|
|
95
|
+
# @return [Array<String>] ordered in-card title candidates (heading → anchor)
|
|
114
96
|
def title_candidates
|
|
115
97
|
[].tap do |candidates|
|
|
116
|
-
add_title_candidate!(candidates,
|
|
117
|
-
add_title_candidate!(candidates, @selected_anchor
|
|
118
|
-
next unless candidates.empty? && @fallback_anchorless && @selected_anchor.nil?
|
|
119
|
-
|
|
120
|
-
add_title_candidate!(candidates, first_nonempty_own_text)
|
|
98
|
+
add_title_candidate!(candidates, heading_title_text)
|
|
99
|
+
add_title_candidate!(candidates, title_pieces.headline_text(@selected_anchor))
|
|
121
100
|
end
|
|
122
101
|
end
|
|
123
102
|
|
|
@@ -131,14 +110,13 @@ module Html2rss
|
|
|
131
110
|
candidates << stripped
|
|
132
111
|
end
|
|
133
112
|
|
|
113
|
+
# @return [String, nil] the heading's headline text ({TitlePieces#headline_text})
|
|
114
|
+
def heading_title_text = title_pieces.headline_text(heading)
|
|
115
|
+
|
|
134
116
|
def heading
|
|
135
117
|
@heading ||= begin
|
|
136
118
|
tags = @root.find_all(&:heading?)
|
|
137
|
-
if tags.any?
|
|
138
|
-
select_best_heading(tags)
|
|
139
|
-
elsif @fallback_anchorless && @selected_anchor.nil?
|
|
140
|
-
fallback_heading
|
|
141
|
-
end
|
|
119
|
+
select_best_heading(tags) if tags.any?
|
|
142
120
|
end
|
|
143
121
|
end
|
|
144
122
|
|
|
@@ -155,42 +133,13 @@ module Html2rss
|
|
|
155
133
|
best
|
|
156
134
|
end
|
|
157
135
|
|
|
158
|
-
def fallback_heading
|
|
159
|
-
@root.find do |n|
|
|
160
|
-
next true if FALLBACK_HEADING_NAMES.include?(n.name)
|
|
161
|
-
|
|
162
|
-
n.attrs.class_names.any? { |c| c.match?(/title|font-bold|font-semibold/) } &&
|
|
163
|
-
!n.visible_text.to_s.strip.empty?
|
|
164
|
-
end
|
|
165
|
-
end
|
|
166
|
-
|
|
167
|
-
def kicker_node # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
168
|
-
@kicker_node ||= begin
|
|
169
|
-
node = @root.find do |n|
|
|
170
|
-
n.attrs.raw.key?('data-tb-kicker') ||
|
|
171
|
-
n.attrs.class_names.any? { |c| c.match?(KICKER_CLASS_PATTERN) }
|
|
172
|
-
end
|
|
173
|
-
node && heading && (node.equal?(heading) || descendant_of?(node, heading)) ? nil : node
|
|
174
|
-
end
|
|
175
|
-
end
|
|
176
|
-
|
|
177
|
-
def descendant_of?(child, ancestor)
|
|
178
|
-
return true if child.equal?(ancestor)
|
|
179
|
-
|
|
180
|
-
if (index = SST::Index.for_node(child))
|
|
181
|
-
return index.descendant_of?(child, ancestor)
|
|
182
|
-
end
|
|
183
|
-
|
|
184
|
-
!ancestor.find { |d| d.equal?(child) }.nil?
|
|
185
|
-
end
|
|
186
|
-
|
|
187
136
|
def leftover_lines
|
|
188
137
|
leftover_lines_from(@root)
|
|
189
138
|
end
|
|
190
139
|
|
|
191
140
|
def leftover_lines_from(node)
|
|
192
141
|
times = node.find_all { |n| n.name == :time }
|
|
193
|
-
exclude = [heading, @selected_anchor,
|
|
142
|
+
exclude = [heading, @selected_anchor, title_pieces.kicker, *times].compact
|
|
194
143
|
ArticleRules::Description.lines_from(node.visible_text(exclude:))
|
|
195
144
|
end
|
|
196
145
|
|
|
@@ -234,7 +183,7 @@ module Html2rss
|
|
|
234
183
|
CardWalk.thin_wrapper?(
|
|
235
184
|
children: node.children,
|
|
236
185
|
item: @root,
|
|
237
|
-
descendant_of: ->(item, child) {
|
|
186
|
+
descendant_of: ->(item, child) { SST::Index.descendant_or_self?(item, child) }
|
|
238
187
|
)
|
|
239
188
|
end
|
|
240
189
|
|
|
@@ -253,48 +202,15 @@ module Html2rss
|
|
|
253
202
|
SST::Index.for_node(@root)&.parent_of(@root)
|
|
254
203
|
end
|
|
255
204
|
|
|
256
|
-
|
|
257
|
-
@generate_id ||= begin
|
|
258
|
-
id_from_dom = parse_id_from_dom
|
|
259
|
-
if id_from_dom
|
|
260
|
-
id_from_dom
|
|
261
|
-
else
|
|
262
|
-
heading_text = heading&.visible_text
|
|
263
|
-
heading_text = first_nonempty_own_text if heading_text.to_s.strip.empty? && @fallback_anchorless
|
|
264
|
-
if heading_text && !heading_text.strip.empty?
|
|
265
|
-
generate_slug(heading_text)
|
|
266
|
-
elsif @fallback_anchorless
|
|
267
|
-
text = @root.visible_text.to_s.strip
|
|
268
|
-
Zlib.crc32(text).to_s(36) unless text.empty?
|
|
269
|
-
end
|
|
270
|
-
end
|
|
271
|
-
end
|
|
272
|
-
end
|
|
273
|
-
|
|
205
|
+
# DOM id of the card, else the link's path or query.
|
|
274
206
|
def parse_id_from_dom
|
|
275
|
-
|
|
207
|
+
url = extract_url
|
|
276
208
|
nested = @root.find { |n| n.attrs.id }
|
|
277
|
-
|
|
278
|
-
if @selected_anchor
|
|
279
|
-
url = extract_url
|
|
280
|
-
candidates += [url&.path, url&.query]
|
|
281
|
-
end
|
|
282
|
-
candidates.compact.reject(&:empty?).first
|
|
283
|
-
end
|
|
284
|
-
|
|
285
|
-
def generate_slug(text)
|
|
286
|
-
slug = text.downcase.gsub(/[^a-z0-9]+/, '-')
|
|
287
|
-
slug = slug[1..] if slug.start_with?('-')
|
|
288
|
-
slug = slug[0..-2] if slug.end_with?('-')
|
|
289
|
-
slug unless slug.empty?
|
|
290
|
-
end
|
|
291
|
-
|
|
292
|
-
def first_nonempty_own_text
|
|
293
|
-
@root.find { |n| !n.own_text.to_s.strip.empty? }&.own_text&.strip
|
|
209
|
+
[@root.attrs.id, nested&.attrs&.id, url&.path, url&.query].compact.reject(&:empty?).first
|
|
294
210
|
end
|
|
295
211
|
|
|
296
212
|
def extract_image
|
|
297
|
-
img_src = image_from_srcset || image_from_img || image_from_style
|
|
213
|
+
img_src = image_from_srcset || image_from_img || image_from_style || image_from_poster
|
|
298
214
|
Url.from_relative(img_src, @base_url) if img_src
|
|
299
215
|
end
|
|
300
216
|
|
|
@@ -302,6 +218,12 @@ module Html2rss
|
|
|
302
218
|
@root.find { |n| n.image? && n.attrs.src && !n.attrs.src.start_with?('data:') }&.attrs&.src
|
|
303
219
|
end
|
|
304
220
|
|
|
221
|
+
# A video's poster frame: the last resort, when the card has no other image.
|
|
222
|
+
def image_from_poster
|
|
223
|
+
posters = @root.find_all { |n| n.name == :video }.map { |n| n.attrs.raw['poster'] }
|
|
224
|
+
posters.find { |src| ArticleRules::Image.remote_src?(src) }
|
|
225
|
+
end
|
|
226
|
+
|
|
305
227
|
def image_from_srcset
|
|
306
228
|
srcsets = @root.find_all { |n| n.attrs.srcset }.map { |n| n.attrs.srcset }
|
|
307
229
|
ArticleRules::Image.largest_from_srcsets(srcsets)
|
|
@@ -317,30 +239,18 @@ module Html2rss
|
|
|
317
239
|
ArticleRules::Date.earliest(datetimes, leftover_lines: lines, time_zone: @time_zone)
|
|
318
240
|
end
|
|
319
241
|
|
|
320
|
-
def extract_enclosures
|
|
321
|
-
@root.find_all { |n| enclosure_node?(n) }.filter_map { |element| enclosure_from(element) }
|
|
322
|
-
end
|
|
242
|
+
def extract_enclosures = ArticleRules::Enclosure.unique(collect_enclosures(@root, nil, []))
|
|
323
243
|
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
case element.name
|
|
335
|
-
when :img
|
|
336
|
-
ArticleRules::Enclosure.from_image(element.attrs.src, @base_url)
|
|
337
|
-
when :video, :audio, :source
|
|
338
|
-
ArticleRules::Enclosure.from_media(element.attrs.src, element.attrs.type, @base_url)
|
|
339
|
-
when :iframe
|
|
340
|
-
ArticleRules::Enclosure.from_iframe(element.attrs.src, @base_url)
|
|
341
|
-
when :a
|
|
342
|
-
ArticleRules::Enclosure.from_anchor(element.attrs.href, @base_url)
|
|
343
|
-
end
|
|
244
|
+
# Depth-first in document order, carrying the nearest player ancestor's name for +source+ elements.
|
|
245
|
+
def collect_enclosures(node, player, found)
|
|
246
|
+
name = node.name.name
|
|
247
|
+
attrs = node.attrs
|
|
248
|
+
enclosure = ArticleRules::Enclosure.from_element(name, @base_url, src: attrs.src, href: attrs.href,
|
|
249
|
+
type: attrs.type) { player }
|
|
250
|
+
found << enclosure if enclosure
|
|
251
|
+
player = name if ArticleRules::Enclosure::MEDIA_ELEMENTS.include?(name)
|
|
252
|
+
node.children.each { |child| collect_enclosures(child, player, found) }
|
|
253
|
+
found
|
|
344
254
|
end
|
|
345
255
|
|
|
346
256
|
def extract_categories(title, root: @root)
|
|
@@ -357,12 +267,12 @@ module Html2rss
|
|
|
357
267
|
end
|
|
358
268
|
|
|
359
269
|
def category_candidate?(node)
|
|
360
|
-
ArticleRules::Category.class_match?(node.attrs.
|
|
361
|
-
node.attrs.raw.
|
|
270
|
+
ArticleRules::Category.class_match?(node.attrs.class_names) ||
|
|
271
|
+
node.attrs.raw.any? { |k, _| ArticleRules::Category.attr_name_match?(k) }
|
|
362
272
|
end
|
|
363
273
|
|
|
364
274
|
def category_text_node?(element, root)
|
|
365
|
-
!element.equal?(root) && ArticleRules::Category.class_match?(element.attrs.
|
|
275
|
+
!element.equal?(root) && ArticleRules::Category.class_match?(element.attrs.class_names)
|
|
366
276
|
end
|
|
367
277
|
|
|
368
278
|
def add_category_text!(categories, element, title:)
|