html2rss 0.33.0 → 0.34.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +4 -2
  3. data/lib/html2rss/article/deduplicator.rb +5 -2
  4. data/lib/html2rss/article/enclosure.rb +185 -21
  5. data/lib/html2rss/article/title.rb +203 -0
  6. data/lib/html2rss/article.rb +79 -13
  7. data/lib/html2rss/auto_source/README.md +16 -8
  8. data/lib/html2rss/auto_source/cleanup.rb +126 -50
  9. data/lib/html2rss/auto_source/corroboration.rb +167 -0
  10. data/lib/html2rss/auto_source/page_links/anchor_claims.rb +82 -0
  11. data/lib/html2rss/auto_source/page_links/card_units.rb +77 -0
  12. data/lib/html2rss/auto_source/page_links/label_binding.rb +79 -0
  13. data/lib/html2rss/auto_source/page_links/sponsored_cards.rb +147 -0
  14. data/lib/html2rss/auto_source/page_links.rb +164 -0
  15. data/lib/html2rss/auto_source/scraper/html.rb +11 -39
  16. data/lib/html2rss/auto_source/scraper/json_document.rb +1 -3
  17. data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +119 -42
  18. data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +5 -16
  19. data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +2 -10
  20. data/lib/html2rss/auto_source/scraper/json_state.rb +2 -3
  21. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +6 -14
  22. data/lib/html2rss/auto_source/scraper/microdata.rb +6 -4
  23. data/lib/html2rss/auto_source/scraper/microformats2.rb +1 -3
  24. data/lib/html2rss/auto_source/scraper/native_feed.rb +27 -17
  25. data/lib/html2rss/auto_source/scraper/schema/item_list.rb +5 -3
  26. data/lib/html2rss/auto_source/scraper/schema/thing.rb +68 -27
  27. data/lib/html2rss/auto_source/scraper/schema.rb +5 -8
  28. data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb +3 -9
  29. data/lib/html2rss/auto_source/scraper/semantic_html.rb +1 -5
  30. data/lib/html2rss/auto_source/scraper/sitemap.rb +4 -3
  31. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -3
  32. data/lib/html2rss/auto_source/scraper/xhr_articles.rb +1 -3
  33. data/lib/html2rss/auto_source/scraper.rb +10 -2
  34. data/lib/html2rss/auto_source/segment.rb +19 -5
  35. data/lib/html2rss/auto_source/segmenter/cluster.rb +13 -11
  36. data/lib/html2rss/auto_source/segmenter/list.rb +13 -5
  37. data/lib/html2rss/auto_source/segmenter/primary_link.rb +29 -4
  38. data/lib/html2rss/auto_source/segmenter/semantic.rb +17 -6
  39. data/lib/html2rss/auto_source/segmenter.rb +4 -8
  40. data/lib/html2rss/auto_source.rb +81 -26
  41. data/lib/html2rss/batch.rb +2 -0
  42. data/lib/html2rss/capture/README.md +5 -2
  43. data/lib/html2rss/capture.rb +44 -32
  44. data/lib/html2rss/channel.rb +30 -12
  45. data/lib/html2rss/cli.rb +22 -11
  46. data/lib/html2rss/config/auto_source_contract.rb +52 -45
  47. data/lib/html2rss/config/issue_mapper.rb +4 -1
  48. data/lib/html2rss/config/request_controls.rb +6 -3
  49. data/lib/html2rss/config/schema.rb +15 -5
  50. data/lib/html2rss/config/selectors_validator.rb +2 -2
  51. data/lib/html2rss/config/validation_issue.rb +4 -1
  52. data/lib/html2rss/config/validator.rb +11 -0
  53. data/lib/html2rss/config.rb +53 -27
  54. data/lib/html2rss/contract_error.rb +9 -0
  55. data/lib/html2rss/defaults.rb +21 -23
  56. data/lib/html2rss/doctor/botasaurus.rb +7 -3
  57. data/lib/html2rss/error.rb +10 -0
  58. data/lib/html2rss/error_kind.rb +39 -0
  59. data/lib/html2rss/feed_builder/item_presentation.rb +22 -9
  60. data/lib/html2rss/feed_builder/json_feed/item.rb +2 -1
  61. data/lib/html2rss/feed_builder/json_feed.rb +2 -2
  62. data/lib/html2rss/feed_builder/rss/stylesheet.rb +3 -3
  63. data/lib/html2rss/feed_builder/rss.rb +49 -6
  64. data/lib/html2rss/feed_builder.rb +2 -1
  65. data/lib/html2rss/feed_pipeline/README.md +1 -1
  66. data/lib/html2rss/feed_pipeline/auto_fallback.rb +13 -31
  67. data/lib/html2rss/feed_pipeline/strategy_plan.rb +3 -3
  68. data/lib/html2rss/feed_pipeline.rb +40 -27
  69. data/lib/html2rss/feed_resolution/README.md +1 -1
  70. data/lib/html2rss/feed_resolution/candidate_generator.rb +3 -4
  71. data/lib/html2rss/feed_resolution/policy.rb +1 -1
  72. data/lib/html2rss/feed_resolution/probe.rb +3 -1
  73. data/lib/html2rss/feed_resolution.rb +11 -12
  74. data/lib/html2rss/feed_result.rb +9 -5
  75. data/lib/html2rss/hash_util.rb +3 -3
  76. data/lib/html2rss/html/article_extractor/category_extractor.rb +1 -1
  77. data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +12 -31
  78. data/lib/html2rss/html/article_extractor/heading_extractor.rb +2 -3
  79. data/lib/html2rss/html/article_extractor/id_generator.rb +6 -9
  80. data/lib/html2rss/html/article_extractor/image_extractor.rb +9 -1
  81. data/lib/html2rss/html/article_extractor.rb +19 -47
  82. data/lib/html2rss/html/article_rules/category.rb +3 -3
  83. data/lib/html2rss/html/article_rules/description.rb +6 -15
  84. data/lib/html2rss/html/article_rules/enclosure.rb +70 -42
  85. data/lib/html2rss/html/article_rules/image.rb +30 -16
  86. data/lib/html2rss/html/feed_link.rb +7 -9
  87. data/lib/html2rss/html/navigator/text_extractor.rb +26 -16
  88. data/lib/html2rss/html/navigator.rb +39 -9
  89. data/lib/html2rss/html/probe.rb +4 -0
  90. data/lib/html2rss/html/rendering/description_builder.rb +46 -32
  91. data/lib/html2rss/html/rendering.rb +2 -9
  92. data/lib/html2rss/html/sst_article_extractor.rb +55 -145
  93. data/lib/html2rss/html/title_pieces/kicker.rb +147 -0
  94. data/lib/html2rss/html/title_pieces/nodes.rb +107 -0
  95. data/lib/html2rss/html/title_pieces/nokogiri_nodes.rb +70 -0
  96. data/lib/html2rss/html/title_pieces/sst_nodes.rb +71 -0
  97. data/lib/html2rss/html/title_pieces.rb +169 -0
  98. data/lib/html2rss/html/visibility.rb +56 -0
  99. data/lib/html2rss/html.rb +2 -0
  100. data/lib/html2rss/link_destination/destination_facts.rb +17 -1
  101. data/lib/html2rss/link_destination/noise_policy.rb +11 -10
  102. data/lib/html2rss/link_destination/path_classifier.rb +100 -9
  103. data/lib/html2rss/link_destination/sponsorship/label_text.rb +82 -0
  104. data/lib/html2rss/link_destination/sponsorship.rb +184 -0
  105. data/lib/html2rss/link_destination/text_classifier.rb +104 -8
  106. data/lib/html2rss/mcp/README.md +13 -1
  107. data/lib/html2rss/mcp/config_argument.rb +5 -2
  108. data/lib/html2rss/mcp/contract.rb +13 -4
  109. data/lib/html2rss/mcp/outcome.rb +14 -12
  110. data/lib/html2rss/mcp/runtime.rb +39 -2
  111. data/lib/html2rss/mcp/server.rb +13 -4
  112. data/lib/html2rss/page_recon/README.md +1 -1
  113. data/lib/html2rss/page_recon/diagnostics.rb +2 -0
  114. data/lib/html2rss/recon.rb +10 -12
  115. data/lib/html2rss/request_service/blocked_surface.rb +2 -1
  116. data/lib/html2rss/request_service/botasaurus_contract.rb +25 -3
  117. data/lib/html2rss/request_service/botasaurus_strategy.rb +42 -10
  118. data/lib/html2rss/request_service/context.rb +15 -14
  119. data/lib/html2rss/request_service/cookie_jar.rb +89 -0
  120. data/lib/html2rss/request_service/cookie_store.rb +79 -0
  121. data/lib/html2rss/request_service/feed_advertisement.rb +72 -0
  122. data/lib/html2rss/request_service/httpx_strategy.rb +7 -3
  123. data/lib/html2rss/request_service/local_file_strategy.rb +2 -2
  124. data/lib/html2rss/request_service/policy.rb +48 -11
  125. data/lib/html2rss/request_service/replay_breaker.rb +56 -0
  126. data/lib/html2rss/request_service/response.rb +30 -13
  127. data/lib/html2rss/request_service/response_cache.rb +109 -0
  128. data/lib/html2rss/request_service/response_caching.rb +95 -0
  129. data/lib/html2rss/request_service/strategy.rb +5 -11
  130. data/lib/html2rss/request_service.rb +71 -12
  131. data/lib/html2rss/request_session/pager/base.rb +0 -7
  132. data/lib/html2rss/request_session/pager.rb +1 -1
  133. data/lib/html2rss/request_session.rb +6 -2
  134. data/lib/html2rss/scoring/cluster_scorer.rb +1 -1
  135. data/lib/html2rss/scoring/container_assessor.rb +12 -13
  136. data/lib/html2rss/scoring/engine.rb +6 -6
  137. data/lib/html2rss/scoring/observation.rb +2 -2
  138. data/lib/html2rss/selector_candidates.rb +19 -8
  139. data/lib/html2rss/selectors/README.md +1 -1
  140. data/lib/html2rss/selectors/option_spec.rb +4 -1
  141. data/lib/html2rss/selectors/post_processors/base.rb +0 -8
  142. data/lib/html2rss/selectors/post_processors/gsub.rb +7 -7
  143. data/lib/html2rss/selectors/post_processors/html_transformers/keep_playable_media.rb +50 -0
  144. data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +1 -3
  145. data/lib/html2rss/selectors/post_processors/parse_time.rb +3 -3
  146. data/lib/html2rss/selectors/post_processors/parse_uri.rb +1 -1
  147. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +18 -5
  148. data/lib/html2rss/selectors/post_processors/substring.rb +1 -1
  149. data/lib/html2rss/selectors.rb +53 -12
  150. data/lib/html2rss/sst/attrs.rb +2 -2
  151. data/lib/html2rss/sst/document.rb +3 -3
  152. data/lib/html2rss/sst/index.rb +30 -11
  153. data/lib/html2rss/sst/node.rb +33 -26
  154. data/lib/html2rss/sst/normalizer.rb +31 -19
  155. data/lib/html2rss/sst/tags.rb +113 -2
  156. data/lib/html2rss/sst/text.rb +69 -29
  157. data/lib/html2rss/sst.rb +2 -0
  158. data/lib/html2rss/status.rb +6 -3
  159. data/lib/html2rss/surface_category.rb +1 -1
  160. data/lib/html2rss/syndication/README.md +12 -6
  161. data/lib/html2rss/syndication/alternate_links.rb +1 -1
  162. data/lib/html2rss/syndication/channel_facts.rb +87 -0
  163. data/lib/html2rss/syndication/discovery.rb +82 -28
  164. data/lib/html2rss/syndication/feed.rb +20 -0
  165. data/lib/html2rss/syndication/item_media.rb +103 -0
  166. data/lib/html2rss/syndication/parser.rb +75 -51
  167. data/lib/html2rss/test/enhance_audit.rb +2 -2
  168. data/lib/html2rss/test/item_sample.rb +43 -0
  169. data/lib/html2rss/test.rb +42 -14
  170. data/lib/html2rss/url.rb +115 -14
  171. data/lib/html2rss/version.rb +1 -1
  172. data/lib/html2rss/word_count.rb +53 -0
  173. data/lib/html2rss.rb +17 -8
  174. data/schema/html2rss-config.schema.json +56 -41
  175. metadata +31 -4
  176. data/lib/html2rss/html/rendering/media_renderer.rb +0 -39
  177. data/lib/html2rss/html/rendering/pdf_renderer.rb +0 -30
@@ -5,7 +5,9 @@ module Html2rss
5
5
  class Navigator
6
6
  ##
7
7
  # TextExtractor extracts visible text from DOM elements, preserving lists
8
- # and block spacing while sanitizing white spaces.
8
+ # and block spacing while sanitizing white spaces. Same joining rule as {SST::Text}: an inline element
9
+ # embedded in direct text and touching it stays in its word (+Straßenbah<span>n</span> für+), while two
10
+ # adjacent elements and an element opening or closing its parent's text always separate.
9
11
  class TextExtractor
10
12
  # HTML block elements that trigger line breaks or special formatting.
11
13
  BLOCK_TAGS = %w[p div li ul ol h1 h2 h3 h4 h5 h6 tr br].to_set.freeze
@@ -29,19 +31,21 @@ module Html2rss
29
31
 
30
32
  private
31
33
 
34
+ # +last+ is [node, block?] of the previous piece with text.
32
35
  def iterate_children(tag, separator, exclude_nodes)
33
- last = false
34
- tag.children.each_with_object([]) do |c, p|
35
- next if exclude_nodes&.include?(c) || !visible_child?(c)
36
+ last = nil
37
+ tag.children.each_with_object([]) do |c, parts|
38
+ text, block = process_child_node(c, separator, exclude_nodes) if shown?(c, exclude_nodes)
39
+ next if text.to_s.empty?
36
40
 
37
- text, block = process_child_node(c, separator, exclude_nodes)
38
- next if text.empty?
39
-
40
- append_separator!(p, separator, block, last)
41
- (p << text) && (last = block)
41
+ parts << gap(separator, block, last, c) if last
42
+ parts << text
43
+ last = [c, block]
42
44
  end
43
45
  end
44
46
 
47
+ def shown?(child, exclude_nodes) = !exclude_nodes&.include?(child) && visible_child?(child)
48
+
45
49
  def process_child_node(child, separator, exclude_nodes)
46
50
  child_text = get_child_text(child, separator, exclude_nodes)
47
51
  return ['', false] if child_text.empty?
@@ -58,16 +62,22 @@ module Html2rss
58
62
  end
59
63
  end
60
64
 
61
- def append_separator!(parts, separator, is_block, last_was_block)
62
- return if parts.empty?
65
+ def gap(separator, is_block, last, child)
66
+ return (separator == ' ' ? "\n" : separator) if is_block || last.last
63
67
 
64
- parts << if is_block || last_was_block
65
- (separator == ' ' ? "\n" : separator)
66
- else
67
- ' '
68
- end
68
+ touching?(last.first, child) ? '' : ' '
69
69
  end
70
70
 
71
+ # Whether direct text and an inline element meet as adjacent siblings with no whitespace between
72
+ # them. Only an element embedded in direct text (a text sibling on both sides) touches it.
73
+ def touching?(before, after)
74
+ element = before.element? ? before : after
75
+ before.next_sibling == after && embedded?(element) &&
76
+ !before.text.match?(/\s\z/) && !after.text.match?(/\A\s/)
77
+ end
78
+
79
+ def embedded?(element) = element.previous_sibling&.text? && element.next_sibling&.text?
80
+
71
81
  def visible_child?(node)
72
82
  !INVISIBLE_CONTENT_TAGS.include?(node.name) &&
73
83
  !(node.name == 'a' && node['href']&.start_with?('#'))
@@ -5,17 +5,17 @@ module Html2rss
5
5
  ##
6
6
  # Navigator owns DOM chrome recognition and node traversal helpers.
7
7
  class Navigator
8
+ # String forms of {SST::Tags} (the vocabulary owner); no delta today.
8
9
  # Heading tags used to prioritize title extraction and container assessment.
9
- HEADING_TAGS = Set['h1', 'h2', 'h3', 'h4', 'h5', 'h6'].freeze
10
+ HEADING_TAGS = SST::Tags::HEADING_NAMES.to_set(&:to_s).freeze
10
11
 
11
12
  # Element tags that indicate ignored DOM chrome when found in a container path.
12
- IGNORED_CONTAINER_TAGS = Set['nav', 'footer', 'header', 'svg', 'script', 'style'].freeze
13
+ IGNORED_CONTAINER_TAGS = SST::Tags::IGNORED_CONTAINER_NAMES.to_set(&:to_s).freeze
13
14
 
14
- # Ancestor tags that usually indicate navigation/utility regions inside a content container.
15
- UTILITY_LANDMARK_TAGS = Set['nav', 'aside', 'footer', 'menu'].freeze
16
-
17
- # Immediate parent walk stops here — not a usable article card.
18
- CARD_WALK_STOP_TAGS = (UTILITY_LANDMARK_TAGS | IGNORED_CONTAINER_TAGS | Set['html', 'body']).freeze
15
+ # Immediate parent walk stops here — not a usable article card. Utility landmarks
16
+ # (tags and roles, {SST::Tags.utility_landmark?}) stop too, and page-level header/footer
17
+ # (or banner/contentinfo roles), via {usable_card_parent?}.
18
+ CARD_WALK_STOP_TAGS = (IGNORED_CONTAINER_TAGS | Set['html', 'body']).freeze
19
19
 
20
20
  # Inner tags that mean a wrapping <a> is a card, not a span-styled list link.
21
21
  WRAPPING_ANCHOR_CHILD_TAGS = (HEADING_TAGS + %w[p]).freeze
@@ -23,7 +23,7 @@ module Html2rss
23
23
  # Anchor selector used to identify the canonical article link element.
24
24
  MAIN_ANCHOR_SELECTOR = begin
25
25
  buf = +'a[href]:not([href=""])'
26
- %w[# javascript: mailto: tel: file:// sms: data:].each do |prefix|
26
+ SST::Tags::SKIP_HREF_PREFIXES.each do |prefix|
27
27
  buf << %[:not([href^="#{prefix}"])]
28
28
  end
29
29
  buf.freeze
@@ -67,6 +67,7 @@ module Html2rss
67
67
 
68
68
  ##
69
69
  # Immediate parent is a usable article card (not html/body/landmark chrome).
70
+ # A header/footer inside article/section/aside/main belongs to its card.
70
71
  #
71
72
  # @param node [Nokogiri::XML::Node, SST::Node, nil]
72
73
  # @return [Boolean]
@@ -74,7 +75,11 @@ module Html2rss
74
75
  return false unless node
75
76
  return false if node.is_a?(Nokogiri::XML::Document)
76
77
 
77
- !CARD_WALK_STOP_TAGS.include?(Probe.tag(node))
78
+ tag = Probe.tag(node)
79
+ role = node.is_a?(SST::Node) ? node.attrs.raw['role'] : node['role']
80
+ return !page_chrome?(node) if SST::Tags.page_level_chrome?(tag, role)
81
+
82
+ !CARD_WALK_STOP_TAGS.include?(tag) && !SST::Tags.utility_landmark?(tag, role)
78
83
  end
79
84
 
80
85
  ##
@@ -123,6 +128,31 @@ module Html2rss
123
128
  end
124
129
  false
125
130
  end
131
+
132
+ private
133
+
134
+ # Whether a header/footer is page-level chrome. SST nodes defer to the
135
+ # Normalizer's chrome index (unindexed nodes stay chrome); Nokogiri nodes
136
+ # check for a sectioning ancestor.
137
+ #
138
+ # @param node [Nokogiri::XML::Node, SST::Node]
139
+ # @return [Boolean]
140
+ def page_chrome?(node)
141
+ if node.is_a?(SST::Node)
142
+ index = SST::Index.for_node(node)
143
+ return index.nil? || index.ignored_chrome?(node)
144
+ end
145
+
146
+ nokogiri_chrome_scope(node) == :chrome
147
+ end
148
+
149
+ # Folds {SST::Tags.chrome_scope} over a Nokogiri node's ancestors, root first.
150
+ #
151
+ # @param node [Nokogiri::XML::Node]
152
+ # @return [Symbol] +:page+, +:sectioned+, or +:chrome+
153
+ def nokogiri_chrome_scope(node)
154
+ (node.ancestors.reverse << node).reduce(:page) { |scope, el| el.element? ? SST::Tags.chrome_scope(el, scope) : scope }
155
+ end
126
156
  end
127
157
  end
128
158
  end
@@ -15,6 +15,10 @@ module Html2rss
15
15
  APPLICATION_RSS_XML = 'application/rss+xml'
16
16
  # @see #mime_match?
17
17
  APPLICATION_ATOM_XML = 'application/atom+xml'
18
+ # @see #mime_match?
19
+ APPLICATION_RDF_XML = 'application/rdf+xml'
20
+ # Every syndication format advertised by +link[rel=alternate]+ (RSS 2.0, Atom, RSS 1.0 / RDF).
21
+ FEED_MIME_TYPES = [APPLICATION_RSS_XML, APPLICATION_ATOM_XML, APPLICATION_RDF_XML].freeze
18
22
 
19
23
  class << self
20
24
  ##
@@ -3,32 +3,38 @@
3
3
  module Html2rss
4
4
  module Html
5
5
  module Rendering
6
- # Builds a sanitized article description from the base text, title, and optional media.
6
+ # Builds a sanitized article description from the base text, title, and at most one media element.
7
7
  #
8
- # Combines media elements (images, audio, video, PDFs) with sanitized text content
9
- # to create rich RSS descriptions that reveal more scraped information.
8
+ # The media element is the item's RSS enclosure when it is an audio or video file (a player), else the item
9
+ # image; a downloadable document (PDF, ZIP) falls back to the image. Nothing is rendered when the sanitized
10
+ # description already references that file: an +src+, +href+ or +srcset+ URL resolving to the same normalized
11
+ # absolute URL. Those attributes are parsed only when the description contains the file name (+File.basename+
12
+ # of its path).
10
13
  #
11
14
  # @example Basic usage
12
15
  # builder = DescriptionBuilder.new(
13
16
  # base: "Article content",
14
17
  # title: "Article Title",
15
18
  # url: "https://example.com",
16
- # enclosures: [enclosure_object],
19
+ # enclosure: enclosure_object,
17
20
  # image: "https://example.com/image.jpg"
18
21
  # )
19
22
  # description = builder.call
20
23
  class DescriptionBuilder
21
- # Removes the specified pattern from the beginning of the text
22
- # within a given range if the pattern occurs before the range's end.
24
+ # XPath of the attributes whose URLs count as the description already showing a file.
25
+ URL_ATTRIBUTES = './/@src|.//@href|.//@srcset'
26
+
27
+ # Removes the first occurrence of the pattern in the text content (never inside a tag or attribute)
28
+ # if it starts before the range's end. Text after a stray +<+ that opens no tag is left alone.
23
29
  #
24
30
  # @param text [String]
25
- # @param pattern [String]
31
+ # @param pattern [String, nil] nothing is removed without one
26
32
  # @param end_of_range [Integer] Optional, defaults to half the text length
27
33
  # @return [String]
28
- def self.remove_pattern_from_start(text, pattern, end_of_range: (text.size * 0.5).to_i)
29
- return text unless text.is_a?(String) && pattern.is_a?(String) && !pattern.empty?
34
+ def self.remove_pattern_from_start(text, pattern, end_of_range: text.size / 2)
35
+ return text unless pattern
30
36
 
31
- index = text.index(pattern)
37
+ index = text.index(/\A(?:<[^>]*>|[^<])*?\K#{Regexp.escape(pattern)}/)
32
38
  return text if index.nil? || index > end_of_range
33
39
 
34
40
  "#{text[0, index]}#{text[(index + pattern.size)..]}"
@@ -36,52 +42,60 @@ module Html2rss
36
42
 
37
43
  # @param base [String] The base text content for the description
38
44
  # @param title [String] The article title (used for alt text and title removal)
39
- # @param url [String, Html2rss::Url] The article URL (used for sanitization)
40
- # @param enclosures [Array<Html2rss::Article::Enclosure>, nil] Media enclosures
41
- # @param image [String, Html2rss::Url, nil] Fallback image URL
42
- def initialize(base:, title:, url:, enclosures:, image:)
45
+ # @param url [String, Html2rss::Url] The article URL (used for sanitization and URL resolution)
46
+ # @param enclosure [Html2rss::Article::Enclosure, nil] the item's RSS enclosure
47
+ # ({FeedBuilder::ItemPresentation.rss_enclosure_for}); rendered only when audio or video
48
+ # @param image [String, Html2rss::Url, nil] item image URL, rendered when the enclosure is no player
49
+ def initialize(base:, title:, url:, enclosure:, image:)
43
50
  @base = base.to_s
44
51
  @title = title
45
52
  @url = url
46
- @enclosures = Array(enclosures)
53
+ @player = enclosure if enclosure&.media?
47
54
  @image = image
48
55
  end
49
56
 
50
- # Generates the complete description with media and sanitized text.
57
+ # Generates the description: the media element (unless the text already shows it) and the sanitized text.
51
58
  #
52
59
  # @return [String, nil] The complete description or nil if empty
53
60
  def call
54
- fragments = []
55
- fragments.concat(Array(rendered_media))
56
- fragments << processed_base_description
61
+ text = processed_base_description
62
+ file = @player&.url || @image
63
+ media = media_html if file && !(text && shown?(text, Url.from_absolute(file)))
57
64
 
58
- result = fragments.compact.join("\n\n").strip
59
- result.empty? ? nil : result
65
+ parts = [media, text].compact
66
+ parts.join("\n\n") unless parts.empty?
60
67
  end
61
68
 
62
69
  private
63
70
 
64
- def rendered_media
65
- rendered = render_enclosures
66
- return rendered if rendered.any?
67
- return render_fallback_image if @image
71
+ def media_html
72
+ if @player&.video? then VideoRenderer.new(url: @player.url, type: @player.type).to_html
73
+ elsif @player then AudioRenderer.new(url: @player.url, type: @player.type).to_html
74
+ else ImageRenderer.new(url: @image, title: @title).to_html
75
+ end
76
+ end
68
77
 
69
- []
78
+ def shown?(text, file)
79
+ text.include?(File.basename(file.path)) && referenced_urls(text).include?(file.to_s)
70
80
  end
71
81
 
72
- def render_enclosures
73
- @enclosures.filter_map do |enclosure|
74
- MediaRenderer.for(enclosure:, image: @image, title: @title)&.to_html
82
+ def referenced_urls(text)
83
+ Nokogiri::HTML.fragment(text).xpath(URL_ATTRIBUTES).flat_map do |attribute|
84
+ value = attribute.value
85
+ values = attribute.name == 'srcset' ? value.split(',').map { _1[/\S+/] } : [value]
86
+ values.map { resolve(_1) }
75
87
  end
76
88
  end
77
89
 
78
- def render_fallback_image
79
- [MediaRenderer.for(enclosure: nil, image: @image, title: @title)&.to_html]
90
+ def resolve(value)
91
+ Url.from_relative(value, @url).to_s
92
+ rescue Url::InvalidUrlError
93
+ nil
80
94
  end
81
95
 
82
96
  def processed_base_description
83
97
  text = self.class.remove_pattern_from_start(@base, @title)
84
- Html2rss::Selectors::PostProcessors::SanitizeHtml.call(text, @url)
98
+ Selectors::PostProcessors::SanitizeHtml.call(text, @url)
85
99
  end
86
100
  end
87
101
  end
@@ -2,8 +2,8 @@
2
2
 
3
3
  module Html2rss
4
4
  module Html
5
- # Namespace for HTML rendering logic, used to generate rich content such as
6
- # images, audio, video, or embedded documents for feed descriptions.
5
+ # Namespace for HTML rendering logic: the feed item description and its one media element
6
+ # (an image, audio or video player).
7
7
  #
8
8
  # @see Html2rss::Html::Rendering::DescriptionBuilder
9
9
  #
@@ -12,13 +12,6 @@ module Html2rss
12
12
  # url: "https://example.com/image.jpg",
13
13
  # title: "Example"
14
14
  # ).to_html
15
- #
16
- # @example
17
- # Html2rss::Html::Rendering::MediaRenderer.for(
18
- # enclosure: nil,
19
- # image: "https://example.com/image.jpg",
20
- # title: "Example"
21
- # )
22
15
  module Rendering
23
16
  end
24
17
  end
@@ -1,7 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'zlib'
4
-
5
3
  module Html2rss
6
4
  module Html
7
5
  ##
@@ -9,10 +7,9 @@ module Html2rss
9
7
  # Port of Html::ArticleExtractor field logic onto SST::Node.
10
8
  # rubocop:disable-next Metrics/ClassLength -- SST field extractors colocated for parity with ArticleExtractor
11
9
  class SstArticleExtractor
12
- # CSS class tokens that mark kicker / eyebrow text (excluded from titles).
13
- KICKER_CLASS_PATTERN = /kicker|eyebrow|pre-title|pretitle|overline/i
14
- # Inline emphasis tags used as title fallbacks when no heading exists.
15
- FALLBACK_HEADING_NAMES = Set[:strong, :b].freeze
10
+ # CSS class tokens that mark title-like nodes. Broad on purpose: also shared with
11
+ # {Html2rss::SelectorCandidates}, which owns no copy of it.
12
+ TITLE_CLASS_PATTERN = /title|font-bold|font-semibold/
16
13
  # Nested blocks that mean a category node is actually a content container.
17
14
  CATEGORY_CONTAINER_NAMES = Set[:p, :article, :section].freeze
18
15
 
@@ -21,49 +18,50 @@ module Html2rss
21
18
  # @param ranked_or_segment [Scoring::RankedSegment, AutoSource::Segment]
22
19
  # @param base_url [String, Html2rss::Url]
23
20
  # @param scraper [Class, nil]
24
- # @param fallback_anchorless [Boolean]
25
21
  # @param time_zone [String] channel time zone for naive leftover dates
26
- # @return [Html2rss::Article, nil]
27
- def call(ranked_or_segment, base_url:, scraper: nil, fallback_anchorless: false, time_zone: 'UTC')
28
- new(ranked_or_segment, base_url:, scraper:, fallback_anchorless:, time_zone:).call
22
+ # @return [Html2rss::Article, nil] nil when the segment has no primary link (no URL of its own)
23
+ def call(ranked_or_segment, base_url:, scraper: nil, time_zone: 'UTC')
24
+ new(ranked_or_segment, base_url:, scraper:, time_zone:).call
29
25
  end
30
26
  end
31
27
 
32
28
  # @param ranked_or_segment [Scoring::RankedSegment, AutoSource::Segment]
33
29
  # @param base_url [String, Html2rss::Url]
34
30
  # @param scraper [Class, nil]
35
- # @param fallback_anchorless [Boolean]
36
31
  # @param time_zone [String] channel time zone for naive leftover dates
37
- def initialize(ranked_or_segment, base_url:, scraper: nil, fallback_anchorless: false, time_zone: 'UTC')
32
+ def initialize(ranked_or_segment, base_url:, scraper: nil, time_zone: 'UTC')
38
33
  segment = ranked_or_segment.is_a?(Scoring::RankedSegment) ? ranked_or_segment.segment : ranked_or_segment
39
- raise ArgumentError, 'segment is required' unless segment.is_a?(AutoSource::Segment)
34
+ raise InvalidError, 'segment is required' unless segment.is_a?(AutoSource::Segment)
40
35
 
41
36
  @segment = segment
42
37
  @root = segment.root_node
43
38
  @selected_anchor = segment.primary_link
44
39
  @base_url = base_url
45
40
  @scraper = scraper
46
- @fallback_anchorless = fallback_anchorless
47
41
  @time_zone = time_zone
48
42
  end
49
43
 
50
44
  ##
51
45
  # @return [Html2rss::Article, nil]
52
46
  def call # rubocop:disable Metrics/MethodLength
53
- title = extract_title
47
+ return unless @selected_anchor
48
+
49
+ title = title_from_in_card_sources
54
50
  lines = leftover_lines
55
51
  published_at = extract_published_at(lines)
56
52
  root, lines, published_at = parent_card_fields(title, lines, published_at)
57
53
  attrs = {
58
54
  title:,
55
+ kicker: title_pieces.kicker_text,
59
56
  url: extract_url,
60
57
  image: extract_image,
61
58
  description: ArticleRules::Description.from_lines(lines, title:),
62
- id: generate_id,
59
+ id: parse_id_from_dom,
63
60
  published_at:,
64
61
  enclosures: extract_enclosures,
65
62
  categories: extract_categories(title, root:),
66
- scraper: @scraper
63
+ scraper: @scraper,
64
+ compose_title: true
67
65
  }
68
66
  article = Article.new(**attrs)
69
67
  article.valid? ? article : nil
@@ -73,33 +71,17 @@ module Html2rss
73
71
 
74
72
  def extract_url
75
73
  @extract_url ||= begin
76
- href = @selected_anchor&.attrs&.href.to_s
77
- if href.empty?
78
- anchorless_url_fallback
79
- else
80
- Url.from_relative(href.split('#').first.strip, @base_url)
81
- end
74
+ href = @selected_anchor.attrs.href.to_s
75
+ Url.from_relative(href.split('#').first.strip, @base_url) unless href.empty?
82
76
  end
83
77
  end
84
78
 
85
- def anchorless_url_fallback
86
- return unless @fallback_anchorless
87
-
88
- id = generate_id
89
- Url.from_relative("##{id}", @base_url) if id
90
- end
91
-
92
- def extract_title
93
- title_text = title_from_in_card_sources
94
- return unless title_text
79
+ # Kicker and headline text ({TitlePieces}) of this card.
80
+ def title_pieces = @title_pieces ||= TitlePieces.new(@root, heading || @selected_anchor, anchor: @selected_anchor)
95
81
 
96
- kicker = kicker_node&.visible_text.to_s.strip
97
- kicker && !kicker.empty? && !title_text.include?(kicker) ? "#{kicker}: #{title_text}" : title_text
98
- end
99
-
100
- # Prefer a non-junk title. Credit-only cards may blank to nil (description can carry
101
- # the item). Other junk shapes must stay present so Cleanup can hard-reject them —
102
- # blanking slug/CMS/template text would admit titleless cards.
82
+ # Prefer a non-junk headline ({Article} composes it with {#kicker_text}). A credit-only card
83
+ # blanks to nil, and Cleanup drops the untitled item as +blank_title+. Other junk shapes stay
84
+ # present so Cleanup rejects them under their own reason (slug, CMS, template).
103
85
  #
104
86
  # @return [String, nil]
105
87
  def title_from_in_card_sources
@@ -110,14 +92,11 @@ module Html2rss
110
92
  candidates.find { |text| AutoSource::Cleanup.junk_reason(text) != :credit }
111
93
  end
112
94
 
113
- # @return [Array<String>] ordered in-card title candidates (heading → anchor → fallback)
95
+ # @return [Array<String>] ordered in-card title candidates (heading → anchor)
114
96
  def title_candidates
115
97
  [].tap do |candidates|
116
- add_title_candidate!(candidates, heading&.visible_text)
117
- add_title_candidate!(candidates, @selected_anchor&.visible_text)
118
- next unless candidates.empty? && @fallback_anchorless && @selected_anchor.nil?
119
-
120
- add_title_candidate!(candidates, first_nonempty_own_text)
98
+ add_title_candidate!(candidates, heading_title_text)
99
+ add_title_candidate!(candidates, title_pieces.headline_text(@selected_anchor))
121
100
  end
122
101
  end
123
102
 
@@ -131,14 +110,13 @@ module Html2rss
131
110
  candidates << stripped
132
111
  end
133
112
 
113
+ # @return [String, nil] the heading's headline text ({TitlePieces#headline_text})
114
+ def heading_title_text = title_pieces.headline_text(heading)
115
+
134
116
  def heading
135
117
  @heading ||= begin
136
118
  tags = @root.find_all(&:heading?)
137
- if tags.any?
138
- select_best_heading(tags)
139
- elsif @fallback_anchorless && @selected_anchor.nil?
140
- fallback_heading
141
- end
119
+ select_best_heading(tags) if tags.any?
142
120
  end
143
121
  end
144
122
 
@@ -155,42 +133,13 @@ module Html2rss
155
133
  best
156
134
  end
157
135
 
158
- def fallback_heading
159
- @root.find do |n|
160
- next true if FALLBACK_HEADING_NAMES.include?(n.name)
161
-
162
- n.attrs.class_names.any? { |c| c.match?(/title|font-bold|font-semibold/) } &&
163
- !n.visible_text.to_s.strip.empty?
164
- end
165
- end
166
-
167
- def kicker_node # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
168
- @kicker_node ||= begin
169
- node = @root.find do |n|
170
- n.attrs.raw.key?('data-tb-kicker') ||
171
- n.attrs.class_names.any? { |c| c.match?(KICKER_CLASS_PATTERN) }
172
- end
173
- node && heading && (node.equal?(heading) || descendant_of?(node, heading)) ? nil : node
174
- end
175
- end
176
-
177
- def descendant_of?(child, ancestor)
178
- return true if child.equal?(ancestor)
179
-
180
- if (index = SST::Index.for_node(child))
181
- return index.descendant_of?(child, ancestor)
182
- end
183
-
184
- !ancestor.find { |d| d.equal?(child) }.nil?
185
- end
186
-
187
136
  def leftover_lines
188
137
  leftover_lines_from(@root)
189
138
  end
190
139
 
191
140
  def leftover_lines_from(node)
192
141
  times = node.find_all { |n| n.name == :time }
193
- exclude = [heading, @selected_anchor, kicker_node, *times].compact
142
+ exclude = [heading, @selected_anchor, title_pieces.kicker, *times].compact
194
143
  ArticleRules::Description.lines_from(node.visible_text(exclude:))
195
144
  end
196
145
 
@@ -234,7 +183,7 @@ module Html2rss
234
183
  CardWalk.thin_wrapper?(
235
184
  children: node.children,
236
185
  item: @root,
237
- descendant_of: ->(item, child) { descendant_of?(item, child) }
186
+ descendant_of: ->(item, child) { SST::Index.descendant_or_self?(item, child) }
238
187
  )
239
188
  end
240
189
 
@@ -253,48 +202,15 @@ module Html2rss
253
202
  SST::Index.for_node(@root)&.parent_of(@root)
254
203
  end
255
204
 
256
- def generate_id # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
257
- @generate_id ||= begin
258
- id_from_dom = parse_id_from_dom
259
- if id_from_dom
260
- id_from_dom
261
- else
262
- heading_text = heading&.visible_text
263
- heading_text = first_nonempty_own_text if heading_text.to_s.strip.empty? && @fallback_anchorless
264
- if heading_text && !heading_text.strip.empty?
265
- generate_slug(heading_text)
266
- elsif @fallback_anchorless
267
- text = @root.visible_text.to_s.strip
268
- Zlib.crc32(text).to_s(36) unless text.empty?
269
- end
270
- end
271
- end
272
- end
273
-
205
+ # DOM id of the card, else the link's path or query.
274
206
  def parse_id_from_dom
275
- candidates = [@root.attrs.id]
207
+ url = extract_url
276
208
  nested = @root.find { |n| n.attrs.id }
277
- candidates << nested.attrs.id if nested
278
- if @selected_anchor
279
- url = extract_url
280
- candidates += [url&.path, url&.query]
281
- end
282
- candidates.compact.reject(&:empty?).first
283
- end
284
-
285
- def generate_slug(text)
286
- slug = text.downcase.gsub(/[^a-z0-9]+/, '-')
287
- slug = slug[1..] if slug.start_with?('-')
288
- slug = slug[0..-2] if slug.end_with?('-')
289
- slug unless slug.empty?
290
- end
291
-
292
- def first_nonempty_own_text
293
- @root.find { |n| !n.own_text.to_s.strip.empty? }&.own_text&.strip
209
+ [@root.attrs.id, nested&.attrs&.id, url&.path, url&.query].compact.reject(&:empty?).first
294
210
  end
295
211
 
296
212
  def extract_image
297
- img_src = image_from_srcset || image_from_img || image_from_style
213
+ img_src = image_from_srcset || image_from_img || image_from_style || image_from_poster
298
214
  Url.from_relative(img_src, @base_url) if img_src
299
215
  end
300
216
 
@@ -302,6 +218,12 @@ module Html2rss
302
218
  @root.find { |n| n.image? && n.attrs.src && !n.attrs.src.start_with?('data:') }&.attrs&.src
303
219
  end
304
220
 
221
+ # A video's poster frame: the last resort, when the card has no other image.
222
+ def image_from_poster
223
+ posters = @root.find_all { |n| n.name == :video }.map { |n| n.attrs.raw['poster'] }
224
+ posters.find { |src| ArticleRules::Image.remote_src?(src) }
225
+ end
226
+
305
227
  def image_from_srcset
306
228
  srcsets = @root.find_all { |n| n.attrs.srcset }.map { |n| n.attrs.srcset }
307
229
  ArticleRules::Image.largest_from_srcsets(srcsets)
@@ -317,30 +239,18 @@ module Html2rss
317
239
  ArticleRules::Date.earliest(datetimes, leftover_lines: lines, time_zone: @time_zone)
318
240
  end
319
241
 
320
- def extract_enclosures
321
- @root.find_all { |n| enclosure_node?(n) }.filter_map { |element| enclosure_from(element) }
322
- end
242
+ def extract_enclosures = ArticleRules::Enclosure.unique(collect_enclosures(@root, nil, []))
323
243
 
324
- def enclosure_node?(node)
325
- case node.name
326
- when :img then node.attrs.src && !node.attrs.src.start_with?('data:')
327
- when :source, :audio, :video, :iframe then node.attrs.src
328
- when :a then ArticleRules::Enclosure.archive_href?(node.attrs.href)
329
- else false
330
- end
331
- end
332
-
333
- def enclosure_from(element)
334
- case element.name
335
- when :img
336
- ArticleRules::Enclosure.from_image(element.attrs.src, @base_url)
337
- when :video, :audio, :source
338
- ArticleRules::Enclosure.from_media(element.attrs.src, element.attrs.type, @base_url)
339
- when :iframe
340
- ArticleRules::Enclosure.from_iframe(element.attrs.src, @base_url)
341
- when :a
342
- ArticleRules::Enclosure.from_anchor(element.attrs.href, @base_url)
343
- end
244
+ # Depth-first in document order, carrying the nearest player ancestor's name for +source+ elements.
245
+ def collect_enclosures(node, player, found)
246
+ name = node.name.name
247
+ attrs = node.attrs
248
+ enclosure = ArticleRules::Enclosure.from_element(name, @base_url, src: attrs.src, href: attrs.href,
249
+ type: attrs.type) { player }
250
+ found << enclosure if enclosure
251
+ player = name if ArticleRules::Enclosure::MEDIA_ELEMENTS.include?(name)
252
+ node.children.each { |child| collect_enclosures(child, player, found) }
253
+ found
344
254
  end
345
255
 
346
256
  def extract_categories(title, root: @root)
@@ -357,12 +267,12 @@ module Html2rss
357
267
  end
358
268
 
359
269
  def category_candidate?(node)
360
- ArticleRules::Category.class_match?(node.attrs.class_attr) ||
361
- node.attrs.raw.keys.any? { |k| ArticleRules::Category.attr_name_match?(k) }
270
+ ArticleRules::Category.class_match?(node.attrs.class_names) ||
271
+ node.attrs.raw.any? { |k, _| ArticleRules::Category.attr_name_match?(k) }
362
272
  end
363
273
 
364
274
  def category_text_node?(element, root)
365
- !element.equal?(root) && ArticleRules::Category.class_match?(element.attrs.class_attr)
275
+ !element.equal?(root) && ArticleRules::Category.class_match?(element.attrs.class_names)
366
276
  end
367
277
 
368
278
  def add_category_text!(categories, element, title:)