html2rss 0.32.4 → 0.34.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +4 -2
  3. data/lib/html2rss/article/deduplicator.rb +5 -2
  4. data/lib/html2rss/article/enclosure.rb +185 -25
  5. data/lib/html2rss/article/title.rb +203 -0
  6. data/lib/html2rss/article.rb +79 -13
  7. data/lib/html2rss/auto_source/README.md +16 -8
  8. data/lib/html2rss/auto_source/cleanup.rb +127 -53
  9. data/lib/html2rss/auto_source/corroboration.rb +167 -0
  10. data/lib/html2rss/auto_source/page_links/anchor_claims.rb +82 -0
  11. data/lib/html2rss/auto_source/page_links/card_units.rb +77 -0
  12. data/lib/html2rss/auto_source/page_links/label_binding.rb +79 -0
  13. data/lib/html2rss/auto_source/page_links/sponsored_cards.rb +147 -0
  14. data/lib/html2rss/auto_source/page_links.rb +164 -0
  15. data/lib/html2rss/auto_source/scraper/html.rb +11 -45
  16. data/lib/html2rss/auto_source/scraper/json_document.rb +3 -6
  17. data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +202 -125
  18. data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +27 -38
  19. data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +123 -123
  20. data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +27 -35
  21. data/lib/html2rss/auto_source/scraper/json_state.rb +4 -9
  22. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +27 -5
  23. data/lib/html2rss/auto_source/scraper/microdata.rb +233 -248
  24. data/lib/html2rss/auto_source/scraper/microformats2.rb +10 -4
  25. data/lib/html2rss/auto_source/scraper/native_feed.rb +28 -18
  26. data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +7 -2
  27. data/lib/html2rss/auto_source/scraper/schema/item_list.rb +5 -3
  28. data/lib/html2rss/auto_source/scraper/schema/thing.rb +140 -22
  29. data/lib/html2rss/auto_source/scraper/schema.rb +52 -8
  30. data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb +3 -9
  31. data/lib/html2rss/auto_source/scraper/semantic_html.rb +12 -20
  32. data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +1 -4
  33. data/lib/html2rss/auto_source/scraper/sitemap.rb +7 -14
  34. data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +2 -2
  35. data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb +1 -3
  36. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +7 -7
  37. data/lib/html2rss/auto_source/scraper/xhr_articles.rb +2 -4
  38. data/lib/html2rss/auto_source/scraper.rb +10 -13
  39. data/lib/html2rss/auto_source/segment.rb +19 -5
  40. data/lib/html2rss/auto_source/segmenter/cluster.rb +18 -22
  41. data/lib/html2rss/auto_source/segmenter/list.rb +35 -59
  42. data/lib/html2rss/auto_source/segmenter/primary_link.rb +29 -4
  43. data/lib/html2rss/auto_source/segmenter/semantic.rb +23 -19
  44. data/lib/html2rss/auto_source/segmenter.rb +4 -8
  45. data/lib/html2rss/auto_source.rb +81 -26
  46. data/lib/html2rss/batch.rb +24 -32
  47. data/lib/html2rss/capture/README.md +6 -3
  48. data/lib/html2rss/capture.rb +51 -59
  49. data/lib/html2rss/channel.rb +30 -12
  50. data/lib/html2rss/cli/render.rb +97 -96
  51. data/lib/html2rss/cli/validate.rb +2 -4
  52. data/lib/html2rss/cli.rb +55 -38
  53. data/lib/html2rss/config/auto_source_contract.rb +52 -45
  54. data/lib/html2rss/config/issue_mapper.rb +4 -1
  55. data/lib/html2rss/config/request_controls.rb +16 -5
  56. data/lib/html2rss/config/schema.rb +23 -15
  57. data/lib/html2rss/config/selectors_validator.rb +3 -2
  58. data/lib/html2rss/config/validation_issue.rb +4 -1
  59. data/lib/html2rss/config/validator.rb +14 -3
  60. data/lib/html2rss/config.rb +54 -28
  61. data/lib/html2rss/contract_error.rb +9 -0
  62. data/lib/html2rss/defaults.rb +25 -38
  63. data/lib/html2rss/doctor/botasaurus.rb +18 -22
  64. data/lib/html2rss/error.rb +17 -3
  65. data/lib/html2rss/error_kind.rb +39 -0
  66. data/lib/html2rss/feed_builder/item_presentation.rb +22 -9
  67. data/lib/html2rss/feed_builder/json_feed/item.rb +2 -1
  68. data/lib/html2rss/feed_builder/json_feed.rb +2 -2
  69. data/lib/html2rss/feed_builder/rss/stylesheet.rb +3 -3
  70. data/lib/html2rss/feed_builder/rss.rb +50 -7
  71. data/lib/html2rss/feed_builder.rb +2 -1
  72. data/lib/html2rss/feed_pipeline/README.md +1 -1
  73. data/lib/html2rss/feed_pipeline/auto_fallback.rb +16 -34
  74. data/lib/html2rss/feed_pipeline/strategy_plan.rb +3 -3
  75. data/lib/html2rss/feed_pipeline.rb +40 -27
  76. data/lib/html2rss/feed_resolution/README.md +1 -1
  77. data/lib/html2rss/feed_resolution/candidate_generator.rb +6 -8
  78. data/lib/html2rss/feed_resolution/options.rb +3 -7
  79. data/lib/html2rss/feed_resolution/policy.rb +29 -33
  80. data/lib/html2rss/feed_resolution/probe.rb +6 -4
  81. data/lib/html2rss/feed_resolution/scorer.rb +29 -29
  82. data/lib/html2rss/feed_resolution.rb +16 -21
  83. data/lib/html2rss/feed_result.rb +11 -6
  84. data/lib/html2rss/hash_util.rb +91 -90
  85. data/lib/html2rss/html/article_extractor/category_extractor.rb +7 -10
  86. data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
  87. data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +12 -33
  88. data/lib/html2rss/html/article_extractor/heading_extractor.rb +2 -3
  89. data/lib/html2rss/html/article_extractor/id_generator.rb +7 -10
  90. data/lib/html2rss/html/article_extractor/image_extractor.rb +9 -1
  91. data/lib/html2rss/html/article_extractor.rb +19 -47
  92. data/lib/html2rss/html/article_rules/category.rb +3 -3
  93. data/lib/html2rss/html/article_rules/description.rb +6 -15
  94. data/lib/html2rss/html/article_rules/enclosure.rb +70 -42
  95. data/lib/html2rss/html/article_rules/image.rb +30 -16
  96. data/lib/html2rss/html/feed_link.rb +7 -9
  97. data/lib/html2rss/html/navigator/text_extractor.rb +26 -16
  98. data/lib/html2rss/html/navigator.rb +39 -9
  99. data/lib/html2rss/html/probe.rb +55 -51
  100. data/lib/html2rss/html/rendering/description_builder.rb +46 -32
  101. data/lib/html2rss/html/rendering.rb +2 -9
  102. data/lib/html2rss/html/sst_article_extractor.rb +55 -145
  103. data/lib/html2rss/html/title_pieces/kicker.rb +147 -0
  104. data/lib/html2rss/html/title_pieces/nodes.rb +107 -0
  105. data/lib/html2rss/html/title_pieces/nokogiri_nodes.rb +70 -0
  106. data/lib/html2rss/html/title_pieces/sst_nodes.rb +71 -0
  107. data/lib/html2rss/html/title_pieces.rb +169 -0
  108. data/lib/html2rss/html/visibility.rb +56 -0
  109. data/lib/html2rss/html.rb +2 -0
  110. data/lib/html2rss/link_destination/destination_facts.rb +17 -1
  111. data/lib/html2rss/link_destination/noise_policy.rb +11 -10
  112. data/lib/html2rss/link_destination/path_classifier.rb +111 -42
  113. data/lib/html2rss/link_destination/sponsorship/label_text.rb +82 -0
  114. data/lib/html2rss/link_destination/sponsorship.rb +184 -0
  115. data/lib/html2rss/link_destination/text_classifier.rb +104 -8
  116. data/lib/html2rss/mcp/README.md +13 -1
  117. data/lib/html2rss/mcp/config_argument.rb +14 -4
  118. data/lib/html2rss/mcp/contract.rb +20 -9
  119. data/lib/html2rss/mcp/outcome.rb +18 -17
  120. data/lib/html2rss/mcp/runtime.rb +53 -23
  121. data/lib/html2rss/mcp/server/tools.rb +9 -9
  122. data/lib/html2rss/mcp/server.rb +13 -4
  123. data/lib/html2rss/option.rb +48 -0
  124. data/lib/html2rss/page_recon/README.md +1 -1
  125. data/lib/html2rss/page_recon/diagnostics.rb +9 -9
  126. data/lib/html2rss/page_recon.rb +0 -4
  127. data/lib/html2rss/recon.rb +19 -23
  128. data/lib/html2rss/request_service/blocked_surface.rb +5 -8
  129. data/lib/html2rss/request_service/botasaurus_contract.rb +25 -3
  130. data/lib/html2rss/request_service/botasaurus_strategy.rb +44 -12
  131. data/lib/html2rss/request_service/budget.rb +3 -24
  132. data/lib/html2rss/request_service/compressed_body.rb +36 -48
  133. data/lib/html2rss/request_service/context.rb +16 -17
  134. data/lib/html2rss/request_service/cookie_jar.rb +89 -0
  135. data/lib/html2rss/request_service/cookie_store.rb +79 -0
  136. data/lib/html2rss/request_service/feed_advertisement.rb +72 -0
  137. data/lib/html2rss/request_service/httpx_strategy.rb +7 -3
  138. data/lib/html2rss/request_service/local_file_strategy.rb +2 -3
  139. data/lib/html2rss/request_service/network_guard.rb +3 -2
  140. data/lib/html2rss/request_service/policy.rb +55 -36
  141. data/lib/html2rss/request_service/replay_breaker.rb +56 -0
  142. data/lib/html2rss/request_service/response.rb +33 -19
  143. data/lib/html2rss/request_service/response_cache.rb +109 -0
  144. data/lib/html2rss/request_service/response_caching.rb +95 -0
  145. data/lib/html2rss/request_service/strategy.rb +5 -11
  146. data/lib/html2rss/request_service.rb +71 -12
  147. data/lib/html2rss/request_session/pager/base.rb +3 -10
  148. data/lib/html2rss/request_session/pager/custom_selector.rb +4 -4
  149. data/lib/html2rss/request_session/pager/offset.rb +1 -7
  150. data/lib/html2rss/request_session/pager/rel_next.rb +1 -1
  151. data/lib/html2rss/request_session/pager/url_template.rb +1 -7
  152. data/lib/html2rss/request_session/pager.rb +1 -1
  153. data/lib/html2rss/request_session.rb +6 -2
  154. data/lib/html2rss/scoring/cluster_scorer.rb +1 -1
  155. data/lib/html2rss/scoring/container_assessor.rb +12 -13
  156. data/lib/html2rss/scoring/engine.rb +10 -38
  157. data/lib/html2rss/scoring/link_resolver.rb +2 -7
  158. data/lib/html2rss/scoring/observation.rb +2 -2
  159. data/lib/html2rss/selector_candidates.rb +19 -8
  160. data/lib/html2rss/selectors/README.md +1 -1
  161. data/lib/html2rss/selectors/extractors/href.rb +2 -4
  162. data/lib/html2rss/selectors/json_xml.rb +33 -34
  163. data/lib/html2rss/selectors/option_spec.rb +4 -1
  164. data/lib/html2rss/selectors/post_processors/base.rb +0 -8
  165. data/lib/html2rss/selectors/post_processors/gsub.rb +9 -10
  166. data/lib/html2rss/selectors/post_processors/html_transformers/keep_playable_media.rb +50 -0
  167. data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +1 -3
  168. data/lib/html2rss/selectors/post_processors/parse_time.rb +5 -6
  169. data/lib/html2rss/selectors/post_processors/parse_uri.rb +1 -1
  170. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +18 -5
  171. data/lib/html2rss/selectors/post_processors/substring.rb +1 -1
  172. data/lib/html2rss/selectors/schema_export.rb +60 -60
  173. data/lib/html2rss/selectors.rb +67 -30
  174. data/lib/html2rss/sst/attrs.rb +2 -2
  175. data/lib/html2rss/sst/document.rb +3 -3
  176. data/lib/html2rss/sst/index.rb +30 -11
  177. data/lib/html2rss/sst/node.rb +42 -29
  178. data/lib/html2rss/sst/normalizer.rb +47 -34
  179. data/lib/html2rss/sst/tags.rb +113 -2
  180. data/lib/html2rss/sst/text.rb +94 -58
  181. data/lib/html2rss/sst.rb +2 -0
  182. data/lib/html2rss/status.rb +27 -31
  183. data/lib/html2rss/surface_category.rb +1 -1
  184. data/lib/html2rss/syndication/README.md +12 -6
  185. data/lib/html2rss/syndication/alternate_links.rb +39 -0
  186. data/lib/html2rss/syndication/channel_facts.rb +87 -0
  187. data/lib/html2rss/syndication/discovery.rb +171 -187
  188. data/lib/html2rss/syndication/feed.rb +20 -0
  189. data/lib/html2rss/syndication/item_media.rb +103 -0
  190. data/lib/html2rss/syndication/parser.rb +117 -116
  191. data/lib/html2rss/test/enhance_audit.rb +15 -17
  192. data/lib/html2rss/test/item_sample.rb +43 -0
  193. data/lib/html2rss/test/policy.rb +9 -11
  194. data/lib/html2rss/test.rb +66 -57
  195. data/lib/html2rss/url.rb +163 -62
  196. data/lib/html2rss/version.rb +1 -1
  197. data/lib/html2rss/word_count.rb +53 -0
  198. data/lib/html2rss.rb +19 -9
  199. data/schema/html2rss-config.schema.json +56 -41
  200. metadata +33 -4
  201. data/lib/html2rss/html/rendering/media_renderer.rb +0 -39
  202. data/lib/html2rss/html/rendering/pdf_renderer.rb +0 -30
@@ -15,67 +15,71 @@ module Html2rss
15
15
  APPLICATION_RSS_XML = 'application/rss+xml'
16
16
  # @see #mime_match?
17
17
  APPLICATION_ATOM_XML = 'application/atom+xml'
18
+ # @see #mime_match?
19
+ APPLICATION_RDF_XML = 'application/rdf+xml'
20
+ # Every syndication format advertised by +link[rel=alternate]+ (RSS 2.0, Atom, RSS 1.0 / RDF).
21
+ FEED_MIME_TYPES = [APPLICATION_RSS_XML, APPLICATION_ATOM_XML, APPLICATION_RDF_XML].freeze
18
22
 
19
- module_function
23
+ class << self
24
+ ##
25
+ # @param string [String, Symbol, #to_s] wire value
26
+ # @return [String] folded comparison key
27
+ def fold(string)
28
+ string.to_s.downcase
29
+ end
20
30
 
21
- ##
22
- # @param string [String, Symbol, #to_s] wire value
23
- # @return [String] folded comparison key
24
- def fold(string)
25
- string.to_s.downcase
26
- end
31
+ ##
32
+ # @param node [Nokogiri::XML::Node]
33
+ # @return [String] folded element name
34
+ def tag(node)
35
+ name = node.name
36
+ return fold(name) unless name.is_a?(String)
37
+ return name if name.ascii_only? && !name.match?(/[A-Z]/)
27
38
 
28
- ##
29
- # @param node [Nokogiri::XML::Node]
30
- # @return [String] folded element name
31
- def tag(node)
32
- name = node.name
33
- return fold(name) unless name.is_a?(String)
34
- return name if name.ascii_only? && !name.match?(/[A-Z]/)
39
+ fold(name)
40
+ end
35
41
 
36
- fold(name)
37
- end
42
+ ##
43
+ # @param type [String, Symbol, #to_s, nil] MIME type attribute value
44
+ # @return [String] media type without parameters, folded
45
+ def mime_base(type)
46
+ raw = type.to_s
47
+ return fold(raw) unless raw.include?(';') || raw.match?(/\A\s|\s\z/)
38
48
 
39
- ##
40
- # @param type [String, Symbol, #to_s, nil] MIME type attribute value
41
- # @return [String] media type without parameters, folded
42
- def mime_base(type)
43
- raw = type.to_s
44
- return fold(raw) unless raw.include?(';') || raw.match?(/\A\s|\s\z/)
49
+ fold(raw.split(';', 2).first.to_s.strip)
50
+ end
45
51
 
46
- fold(raw.split(';', 2).first.to_s.strip)
47
- end
52
+ ##
53
+ # @param actual [String, Symbol, #to_s, nil] observed MIME type
54
+ # @param expected [Array<String>] canonical MIME types to match
55
+ # @return [Boolean]
56
+ def mime_match?(actual, *expected)
57
+ base = mime_base(actual)
58
+ expected.any? { |candidate| base == candidate || base == mime_base(candidate) }
59
+ end
48
60
 
49
- ##
50
- # @param actual [String, Symbol, #to_s, nil] observed MIME type
51
- # @param expected [Array<String>] canonical MIME types to match
52
- # @return [Boolean]
53
- def mime_match?(actual, *expected)
54
- base = mime_base(actual)
55
- expected.any? { |candidate| base == candidate || base == mime_base(candidate) }
56
- end
57
-
58
- ##
59
- # @param doc [Nokogiri::XML::Node]
60
- # @param mime_types [Array<String>] optional MIME filters (folded once per call)
61
- # @return [Array<Nokogiri::XML::Element>]
62
- def scripts(doc, *mime_types)
63
- return doc.css('script') if mime_types.empty?
61
+ ##
62
+ # @param doc [Nokogiri::XML::Node]
63
+ # @param mime_types [Array<String>] optional MIME filters (folded once per call)
64
+ # @return [Array<Nokogiri::XML::Element>]
65
+ def scripts(doc, *mime_types)
66
+ return doc.css('script') if mime_types.empty?
64
67
 
65
- allowed = mime_types.to_set { |mime| mime_base(mime) }
66
- doc.css('script').select { |script| allowed.include?(mime_base(script['type'])) }
67
- end
68
+ allowed = mime_types.to_set { |mime| mime_base(mime) }
69
+ doc.css('script').select { |script| allowed.include?(mime_base(script['type'])) }
70
+ end
68
71
 
69
- ##
70
- # @param doc [Nokogiri::XML::Node]
71
- # @param rel [String] link rel token
72
- # @param mime [String, nil] optional MIME filter
73
- # @return [Array<Nokogiri::XML::Element>]
74
- def alternate_links(doc, rel:, mime: nil)
75
- nodes = doc.css(%(link[rel~="#{rel}"][href]))
76
- return nodes if mime.nil?
72
+ ##
73
+ # @param doc [Nokogiri::XML::Node]
74
+ # @param rel [String] link rel token
75
+ # @param mime [String, nil] optional MIME filter
76
+ # @return [Array<Nokogiri::XML::Element>]
77
+ def alternate_links(doc, rel:, mime: nil)
78
+ nodes = doc.css(%(link[rel~="#{rel}"][href]))
79
+ return nodes if mime.nil?
77
80
 
78
- nodes.select { |node| mime_match?(node['type'], mime) }
81
+ nodes.select { |node| mime_match?(node['type'], mime) }
82
+ end
79
83
  end
80
84
  end
81
85
  end
@@ -3,32 +3,38 @@
3
3
  module Html2rss
4
4
  module Html
5
5
  module Rendering
6
- # Builds a sanitized article description from the base text, title, and optional media.
6
+ # Builds a sanitized article description from the base text, title, and at most one media element.
7
7
  #
8
- # Combines media elements (images, audio, video, PDFs) with sanitized text content
9
- # to create rich RSS descriptions that reveal more scraped information.
8
+ # The media element is the item's RSS enclosure when it is an audio or video file (a player), else the item
9
+ # image; a downloadable document (PDF, ZIP) falls back to the image. Nothing is rendered when the sanitized
10
+ # description already references that file: an +src+, +href+ or +srcset+ URL resolving to the same normalized
11
+ # absolute URL. Those attributes are parsed only when the description contains the file name (+File.basename+
12
+ # of its path).
10
13
  #
11
14
  # @example Basic usage
12
15
  # builder = DescriptionBuilder.new(
13
16
  # base: "Article content",
14
17
  # title: "Article Title",
15
18
  # url: "https://example.com",
16
- # enclosures: [enclosure_object],
19
+ # enclosure: enclosure_object,
17
20
  # image: "https://example.com/image.jpg"
18
21
  # )
19
22
  # description = builder.call
20
23
  class DescriptionBuilder
21
- # Removes the specified pattern from the beginning of the text
22
- # within a given range if the pattern occurs before the range's end.
24
+ # XPath of the attributes whose URLs count as the description already showing a file.
25
+ URL_ATTRIBUTES = './/@src|.//@href|.//@srcset'
26
+
27
+ # Removes the first occurrence of the pattern in the text content (never inside a tag or attribute)
28
+ # if it starts before the range's end. Text after a stray +<+ that opens no tag is left alone.
23
29
  #
24
30
  # @param text [String]
25
- # @param pattern [String]
31
+ # @param pattern [String, nil] nothing is removed without one
26
32
  # @param end_of_range [Integer] Optional, defaults to half the text length
27
33
  # @return [String]
28
- def self.remove_pattern_from_start(text, pattern, end_of_range: (text.size * 0.5).to_i)
29
- return text unless text.is_a?(String) && pattern.is_a?(String) && !pattern.empty?
34
+ def self.remove_pattern_from_start(text, pattern, end_of_range: text.size / 2)
35
+ return text unless pattern
30
36
 
31
- index = text.index(pattern)
37
+ index = text.index(/\A(?:<[^>]*>|[^<])*?\K#{Regexp.escape(pattern)}/)
32
38
  return text if index.nil? || index > end_of_range
33
39
 
34
40
  "#{text[0, index]}#{text[(index + pattern.size)..]}"
@@ -36,52 +42,60 @@ module Html2rss
36
42
 
37
43
  # @param base [String] The base text content for the description
38
44
  # @param title [String] The article title (used for alt text and title removal)
39
- # @param url [String, Html2rss::Url] The article URL (used for sanitization)
40
- # @param enclosures [Array<Html2rss::Article::Enclosure>, nil] Media enclosures
41
- # @param image [String, Html2rss::Url, nil] Fallback image URL
42
- def initialize(base:, title:, url:, enclosures:, image:)
45
+ # @param url [String, Html2rss::Url] The article URL (used for sanitization and URL resolution)
46
+ # @param enclosure [Html2rss::Article::Enclosure, nil] the item's RSS enclosure
47
+ # ({FeedBuilder::ItemPresentation.rss_enclosure_for}); rendered only when audio or video
48
+ # @param image [String, Html2rss::Url, nil] item image URL, rendered when the enclosure is no player
49
+ def initialize(base:, title:, url:, enclosure:, image:)
43
50
  @base = base.to_s
44
51
  @title = title
45
52
  @url = url
46
- @enclosures = Array(enclosures)
53
+ @player = enclosure if enclosure&.media?
47
54
  @image = image
48
55
  end
49
56
 
50
- # Generates the complete description with media and sanitized text.
57
+ # Generates the description: the media element (unless the text already shows it) and the sanitized text.
51
58
  #
52
59
  # @return [String, nil] The complete description or nil if empty
53
60
  def call
54
- fragments = []
55
- fragments.concat(Array(rendered_media))
56
- fragments << processed_base_description
61
+ text = processed_base_description
62
+ file = @player&.url || @image
63
+ media = media_html if file && !(text && shown?(text, Url.from_absolute(file)))
57
64
 
58
- result = fragments.compact.join("\n\n").strip
59
- result.empty? ? nil : result
65
+ parts = [media, text].compact
66
+ parts.join("\n\n") unless parts.empty?
60
67
  end
61
68
 
62
69
  private
63
70
 
64
- def rendered_media
65
- rendered = render_enclosures
66
- return rendered if rendered.any?
67
- return render_fallback_image if @image
71
+ def media_html
72
+ if @player&.video? then VideoRenderer.new(url: @player.url, type: @player.type).to_html
73
+ elsif @player then AudioRenderer.new(url: @player.url, type: @player.type).to_html
74
+ else ImageRenderer.new(url: @image, title: @title).to_html
75
+ end
76
+ end
68
77
 
69
- []
78
+ def shown?(text, file)
79
+ text.include?(File.basename(file.path)) && referenced_urls(text).include?(file.to_s)
70
80
  end
71
81
 
72
- def render_enclosures
73
- @enclosures.filter_map do |enclosure|
74
- MediaRenderer.for(enclosure:, image: @image, title: @title)&.to_html
82
+ def referenced_urls(text)
83
+ Nokogiri::HTML.fragment(text).xpath(URL_ATTRIBUTES).flat_map do |attribute|
84
+ value = attribute.value
85
+ values = attribute.name == 'srcset' ? value.split(',').map { _1[/\S+/] } : [value]
86
+ values.map { resolve(_1) }
75
87
  end
76
88
  end
77
89
 
78
- def render_fallback_image
79
- [MediaRenderer.for(enclosure: nil, image: @image, title: @title)&.to_html]
90
+ def resolve(value)
91
+ Url.from_relative(value, @url).to_s
92
+ rescue Url::InvalidUrlError
93
+ nil
80
94
  end
81
95
 
82
96
  def processed_base_description
83
97
  text = self.class.remove_pattern_from_start(@base, @title)
84
- Html2rss::Selectors::PostProcessors::SanitizeHtml.call(text, @url)
98
+ Selectors::PostProcessors::SanitizeHtml.call(text, @url)
85
99
  end
86
100
  end
87
101
  end
@@ -2,8 +2,8 @@
2
2
 
3
3
  module Html2rss
4
4
  module Html
5
- # Namespace for HTML rendering logic, used to generate rich content such as
6
- # images, audio, video, or embedded documents for feed descriptions.
5
+ # Namespace for HTML rendering logic: the feed item description and its one media element
6
+ # (an image, audio or video player).
7
7
  #
8
8
  # @see Html2rss::Html::Rendering::DescriptionBuilder
9
9
  #
@@ -12,13 +12,6 @@ module Html2rss
12
12
  # url: "https://example.com/image.jpg",
13
13
  # title: "Example"
14
14
  # ).to_html
15
- #
16
- # @example
17
- # Html2rss::Html::Rendering::MediaRenderer.for(
18
- # enclosure: nil,
19
- # image: "https://example.com/image.jpg",
20
- # title: "Example"
21
- # )
22
15
  module Rendering
23
16
  end
24
17
  end
@@ -1,7 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'zlib'
4
-
5
3
  module Html2rss
6
4
  module Html
7
5
  ##
@@ -9,10 +7,9 @@ module Html2rss
9
7
  # Port of Html::ArticleExtractor field logic onto SST::Node.
10
8
  # rubocop:disable-next Metrics/ClassLength -- SST field extractors colocated for parity with ArticleExtractor
11
9
  class SstArticleExtractor
12
- # CSS class tokens that mark kicker / eyebrow text (excluded from titles).
13
- KICKER_CLASS_PATTERN = /kicker|eyebrow|pre-title|pretitle|overline/i
14
- # Inline emphasis tags used as title fallbacks when no heading exists.
15
- FALLBACK_HEADING_NAMES = Set[:strong, :b].freeze
10
+ # CSS class tokens that mark title-like nodes. Broad on purpose: also shared with
11
+ # {Html2rss::SelectorCandidates}, which owns no copy of it.
12
+ TITLE_CLASS_PATTERN = /title|font-bold|font-semibold/
16
13
  # Nested blocks that mean a category node is actually a content container.
17
14
  CATEGORY_CONTAINER_NAMES = Set[:p, :article, :section].freeze
18
15
 
@@ -21,49 +18,50 @@ module Html2rss
21
18
  # @param ranked_or_segment [Scoring::RankedSegment, AutoSource::Segment]
22
19
  # @param base_url [String, Html2rss::Url]
23
20
  # @param scraper [Class, nil]
24
- # @param fallback_anchorless [Boolean]
25
21
  # @param time_zone [String] channel time zone for naive leftover dates
26
- # @return [Html2rss::Article, nil]
27
- def call(ranked_or_segment, base_url:, scraper: nil, fallback_anchorless: false, time_zone: 'UTC')
28
- new(ranked_or_segment, base_url:, scraper:, fallback_anchorless:, time_zone:).call
22
+ # @return [Html2rss::Article, nil] nil when the segment has no primary link (no URL of its own)
23
+ def call(ranked_or_segment, base_url:, scraper: nil, time_zone: 'UTC')
24
+ new(ranked_or_segment, base_url:, scraper:, time_zone:).call
29
25
  end
30
26
  end
31
27
 
32
28
  # @param ranked_or_segment [Scoring::RankedSegment, AutoSource::Segment]
33
29
  # @param base_url [String, Html2rss::Url]
34
30
  # @param scraper [Class, nil]
35
- # @param fallback_anchorless [Boolean]
36
31
  # @param time_zone [String] channel time zone for naive leftover dates
37
- def initialize(ranked_or_segment, base_url:, scraper: nil, fallback_anchorless: false, time_zone: 'UTC')
32
+ def initialize(ranked_or_segment, base_url:, scraper: nil, time_zone: 'UTC')
38
33
  segment = ranked_or_segment.is_a?(Scoring::RankedSegment) ? ranked_or_segment.segment : ranked_or_segment
39
- raise ArgumentError, 'segment is required' unless segment.is_a?(AutoSource::Segment)
34
+ raise InvalidError, 'segment is required' unless segment.is_a?(AutoSource::Segment)
40
35
 
41
36
  @segment = segment
42
37
  @root = segment.root_node
43
38
  @selected_anchor = segment.primary_link
44
39
  @base_url = base_url
45
40
  @scraper = scraper
46
- @fallback_anchorless = fallback_anchorless
47
41
  @time_zone = time_zone
48
42
  end
49
43
 
50
44
  ##
51
45
  # @return [Html2rss::Article, nil]
52
46
  def call # rubocop:disable Metrics/MethodLength
53
- title = extract_title
47
+ return unless @selected_anchor
48
+
49
+ title = title_from_in_card_sources
54
50
  lines = leftover_lines
55
51
  published_at = extract_published_at(lines)
56
52
  root, lines, published_at = parent_card_fields(title, lines, published_at)
57
53
  attrs = {
58
54
  title:,
55
+ kicker: title_pieces.kicker_text,
59
56
  url: extract_url,
60
57
  image: extract_image,
61
58
  description: ArticleRules::Description.from_lines(lines, title:),
62
- id: generate_id,
59
+ id: parse_id_from_dom,
63
60
  published_at:,
64
61
  enclosures: extract_enclosures,
65
62
  categories: extract_categories(title, root:),
66
- scraper: @scraper
63
+ scraper: @scraper,
64
+ compose_title: true
67
65
  }
68
66
  article = Article.new(**attrs)
69
67
  article.valid? ? article : nil
@@ -73,33 +71,17 @@ module Html2rss
73
71
 
74
72
  def extract_url
75
73
  @extract_url ||= begin
76
- href = @selected_anchor&.attrs&.href.to_s
77
- if href.empty?
78
- anchorless_url_fallback
79
- else
80
- Url.from_relative(href.split('#').first.strip, @base_url)
81
- end
74
+ href = @selected_anchor.attrs.href.to_s
75
+ Url.from_relative(href.split('#').first.strip, @base_url) unless href.empty?
82
76
  end
83
77
  end
84
78
 
85
- def anchorless_url_fallback
86
- return unless @fallback_anchorless
87
-
88
- id = generate_id
89
- Url.from_relative("##{id}", @base_url) if id
90
- end
91
-
92
- def extract_title
93
- title_text = title_from_in_card_sources
94
- return unless title_text
79
+ # Kicker and headline text ({TitlePieces}) of this card.
80
+ def title_pieces = @title_pieces ||= TitlePieces.new(@root, heading || @selected_anchor, anchor: @selected_anchor)
95
81
 
96
- kicker = kicker_node&.visible_text.to_s.strip
97
- kicker && !kicker.empty? && !title_text.include?(kicker) ? "#{kicker}: #{title_text}" : title_text
98
- end
99
-
100
- # Prefer a non-junk title. Credit-only cards may blank to nil (description can carry
101
- # the item). Other junk shapes must stay present so Cleanup can hard-reject them —
102
- # blanking slug/CMS/template text would admit titleless cards.
82
+ # Prefer a non-junk headline ({Article} composes it with {#kicker_text}). A credit-only card
83
+ # blanks to nil, and Cleanup drops the untitled item as +blank_title+. Other junk shapes stay
84
+ # present so Cleanup rejects them under their own reason (slug, CMS, template).
103
85
  #
104
86
  # @return [String, nil]
105
87
  def title_from_in_card_sources
@@ -110,14 +92,11 @@ module Html2rss
110
92
  candidates.find { |text| AutoSource::Cleanup.junk_reason(text) != :credit }
111
93
  end
112
94
 
113
- # @return [Array<String>] ordered in-card title candidates (heading → anchor → fallback)
95
+ # @return [Array<String>] ordered in-card title candidates (heading → anchor)
114
96
  def title_candidates
115
97
  [].tap do |candidates|
116
- add_title_candidate!(candidates, heading&.visible_text)
117
- add_title_candidate!(candidates, @selected_anchor&.visible_text)
118
- next unless candidates.empty? && @fallback_anchorless && @selected_anchor.nil?
119
-
120
- add_title_candidate!(candidates, first_nonempty_own_text)
98
+ add_title_candidate!(candidates, heading_title_text)
99
+ add_title_candidate!(candidates, title_pieces.headline_text(@selected_anchor))
121
100
  end
122
101
  end
123
102
 
@@ -131,14 +110,13 @@ module Html2rss
131
110
  candidates << stripped
132
111
  end
133
112
 
113
+ # @return [String, nil] the heading's headline text ({TitlePieces#headline_text})
114
+ def heading_title_text = title_pieces.headline_text(heading)
115
+
134
116
  def heading
135
117
  @heading ||= begin
136
118
  tags = @root.find_all(&:heading?)
137
- if tags.any?
138
- select_best_heading(tags)
139
- elsif @fallback_anchorless && @selected_anchor.nil?
140
- fallback_heading
141
- end
119
+ select_best_heading(tags) if tags.any?
142
120
  end
143
121
  end
144
122
 
@@ -155,42 +133,13 @@ module Html2rss
155
133
  best
156
134
  end
157
135
 
158
- def fallback_heading
159
- @root.find do |n|
160
- next true if FALLBACK_HEADING_NAMES.include?(n.name)
161
-
162
- n.attrs.class_names.any? { |c| c.match?(/title|font-bold|font-semibold/) } &&
163
- !n.visible_text.to_s.strip.empty?
164
- end
165
- end
166
-
167
- def kicker_node # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
168
- @kicker_node ||= begin
169
- node = @root.find do |n|
170
- n.attrs.raw.key?('data-tb-kicker') ||
171
- n.attrs.class_names.any? { |c| c.match?(KICKER_CLASS_PATTERN) }
172
- end
173
- node && heading && (node.equal?(heading) || descendant_of?(node, heading)) ? nil : node
174
- end
175
- end
176
-
177
- def descendant_of?(child, ancestor)
178
- return true if child.equal?(ancestor)
179
-
180
- if (index = SST::Index.for_node(child))
181
- return index.descendant_of?(child, ancestor)
182
- end
183
-
184
- !ancestor.find { |d| d.equal?(child) }.nil?
185
- end
186
-
187
136
  def leftover_lines
188
137
  leftover_lines_from(@root)
189
138
  end
190
139
 
191
140
  def leftover_lines_from(node)
192
141
  times = node.find_all { |n| n.name == :time }
193
- exclude = [heading, @selected_anchor, kicker_node, *times].compact
142
+ exclude = [heading, @selected_anchor, title_pieces.kicker, *times].compact
194
143
  ArticleRules::Description.lines_from(node.visible_text(exclude:))
195
144
  end
196
145
 
@@ -234,7 +183,7 @@ module Html2rss
234
183
  CardWalk.thin_wrapper?(
235
184
  children: node.children,
236
185
  item: @root,
237
- descendant_of: ->(item, child) { descendant_of?(item, child) }
186
+ descendant_of: ->(item, child) { SST::Index.descendant_or_self?(item, child) }
238
187
  )
239
188
  end
240
189
 
@@ -253,48 +202,15 @@ module Html2rss
253
202
  SST::Index.for_node(@root)&.parent_of(@root)
254
203
  end
255
204
 
256
- def generate_id # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
257
- @generate_id ||= begin
258
- id_from_dom = parse_id_from_dom
259
- if id_from_dom
260
- id_from_dom
261
- else
262
- heading_text = heading&.visible_text
263
- heading_text = first_nonempty_own_text if heading_text.to_s.strip.empty? && @fallback_anchorless
264
- if heading_text && !heading_text.strip.empty?
265
- generate_slug(heading_text)
266
- elsif @fallback_anchorless
267
- text = @root.visible_text.to_s.strip
268
- Zlib.crc32(text).to_s(36) unless text.empty?
269
- end
270
- end
271
- end
272
- end
273
-
205
+ # DOM id of the card, else the link's path or query.
274
206
  def parse_id_from_dom
275
- candidates = [@root.attrs.id]
207
+ url = extract_url
276
208
  nested = @root.find { |n| n.attrs.id }
277
- candidates << nested.attrs.id if nested
278
- if @selected_anchor
279
- url = extract_url
280
- candidates += [url&.path, url&.query]
281
- end
282
- candidates.compact.reject(&:empty?).first
283
- end
284
-
285
- def generate_slug(text)
286
- slug = text.downcase.gsub(/[^a-z0-9]+/, '-')
287
- slug = slug[1..] if slug.start_with?('-')
288
- slug = slug[0..-2] if slug.end_with?('-')
289
- slug unless slug.empty?
290
- end
291
-
292
- def first_nonempty_own_text
293
- @root.find { |n| !n.own_text.to_s.strip.empty? }&.own_text&.strip
209
+ [@root.attrs.id, nested&.attrs&.id, url&.path, url&.query].compact.reject(&:empty?).first
294
210
  end
295
211
 
296
212
  def extract_image
297
- img_src = image_from_srcset || image_from_img || image_from_style
213
+ img_src = image_from_srcset || image_from_img || image_from_style || image_from_poster
298
214
  Url.from_relative(img_src, @base_url) if img_src
299
215
  end
300
216
 
@@ -302,6 +218,12 @@ module Html2rss
302
218
  @root.find { |n| n.image? && n.attrs.src && !n.attrs.src.start_with?('data:') }&.attrs&.src
303
219
  end
304
220
 
221
+ # A video's poster frame: the last resort, when the card has no other image.
222
+ def image_from_poster
223
+ posters = @root.find_all { |n| n.name == :video }.map { |n| n.attrs.raw['poster'] }
224
+ posters.find { |src| ArticleRules::Image.remote_src?(src) }
225
+ end
226
+
305
227
  def image_from_srcset
306
228
  srcsets = @root.find_all { |n| n.attrs.srcset }.map { |n| n.attrs.srcset }
307
229
  ArticleRules::Image.largest_from_srcsets(srcsets)
@@ -317,30 +239,18 @@ module Html2rss
317
239
  ArticleRules::Date.earliest(datetimes, leftover_lines: lines, time_zone: @time_zone)
318
240
  end
319
241
 
320
- def extract_enclosures
321
- @root.find_all { |n| enclosure_node?(n) }.filter_map { |element| enclosure_from(element) }
322
- end
242
+ def extract_enclosures = ArticleRules::Enclosure.unique(collect_enclosures(@root, nil, []))
323
243
 
324
- def enclosure_node?(node)
325
- case node.name
326
- when :img then node.attrs.src && !node.attrs.src.start_with?('data:')
327
- when :source, :audio, :video, :iframe then node.attrs.src
328
- when :a then ArticleRules::Enclosure.archive_href?(node.attrs.href)
329
- else false
330
- end
331
- end
332
-
333
- def enclosure_from(element)
334
- case element.name
335
- when :img
336
- ArticleRules::Enclosure.from_image(element.attrs.src, @base_url)
337
- when :video, :audio, :source
338
- ArticleRules::Enclosure.from_media(element.attrs.src, element.attrs.type, @base_url)
339
- when :iframe
340
- ArticleRules::Enclosure.from_iframe(element.attrs.src, @base_url)
341
- when :a
342
- ArticleRules::Enclosure.from_anchor(element.attrs.href, @base_url)
343
- end
244
+ # Depth-first in document order, carrying the nearest player ancestor's name for +source+ elements.
245
+ def collect_enclosures(node, player, found)
246
+ name = node.name.name
247
+ attrs = node.attrs
248
+ enclosure = ArticleRules::Enclosure.from_element(name, @base_url, src: attrs.src, href: attrs.href,
249
+ type: attrs.type) { player }
250
+ found << enclosure if enclosure
251
+ player = name if ArticleRules::Enclosure::MEDIA_ELEMENTS.include?(name)
252
+ node.children.each { |child| collect_enclosures(child, player, found) }
253
+ found
344
254
  end
345
255
 
346
256
  def extract_categories(title, root: @root)
@@ -357,12 +267,12 @@ module Html2rss
357
267
  end
358
268
 
359
269
  def category_candidate?(node)
360
- ArticleRules::Category.class_match?(node.attrs.class_attr) ||
361
- node.attrs.raw.keys.any? { |k| ArticleRules::Category.attr_name_match?(k) }
270
+ ArticleRules::Category.class_match?(node.attrs.class_names) ||
271
+ node.attrs.raw.any? { |k, _| ArticleRules::Category.attr_name_match?(k) }
362
272
  end
363
273
 
364
274
  def category_text_node?(element, root)
365
- !element.equal?(root) && ArticleRules::Category.class_match?(element.attrs.class_attr)
275
+ !element.equal?(root) && ArticleRules::Category.class_match?(element.attrs.class_names)
366
276
  end
367
277
 
368
278
  def add_category_text!(categories, element, title:)