html2rss 0.33.0 → 0.34.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +4 -2
  3. data/lib/html2rss/article/deduplicator.rb +5 -2
  4. data/lib/html2rss/article/enclosure.rb +185 -21
  5. data/lib/html2rss/article/title.rb +203 -0
  6. data/lib/html2rss/article.rb +79 -13
  7. data/lib/html2rss/auto_source/README.md +16 -8
  8. data/lib/html2rss/auto_source/cleanup.rb +126 -50
  9. data/lib/html2rss/auto_source/corroboration.rb +167 -0
  10. data/lib/html2rss/auto_source/page_links/anchor_claims.rb +82 -0
  11. data/lib/html2rss/auto_source/page_links/card_units.rb +77 -0
  12. data/lib/html2rss/auto_source/page_links/label_binding.rb +79 -0
  13. data/lib/html2rss/auto_source/page_links/sponsored_cards.rb +147 -0
  14. data/lib/html2rss/auto_source/page_links.rb +164 -0
  15. data/lib/html2rss/auto_source/scraper/html.rb +11 -39
  16. data/lib/html2rss/auto_source/scraper/json_document.rb +1 -3
  17. data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +119 -42
  18. data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +5 -16
  19. data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +2 -10
  20. data/lib/html2rss/auto_source/scraper/json_state.rb +2 -3
  21. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +6 -14
  22. data/lib/html2rss/auto_source/scraper/microdata.rb +6 -4
  23. data/lib/html2rss/auto_source/scraper/microformats2.rb +1 -3
  24. data/lib/html2rss/auto_source/scraper/native_feed.rb +27 -17
  25. data/lib/html2rss/auto_source/scraper/schema/item_list.rb +5 -3
  26. data/lib/html2rss/auto_source/scraper/schema/thing.rb +68 -27
  27. data/lib/html2rss/auto_source/scraper/schema.rb +5 -8
  28. data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb +3 -9
  29. data/lib/html2rss/auto_source/scraper/semantic_html.rb +1 -5
  30. data/lib/html2rss/auto_source/scraper/sitemap.rb +4 -3
  31. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +3 -3
  32. data/lib/html2rss/auto_source/scraper/xhr_articles.rb +1 -3
  33. data/lib/html2rss/auto_source/scraper.rb +10 -2
  34. data/lib/html2rss/auto_source/segment.rb +19 -5
  35. data/lib/html2rss/auto_source/segmenter/cluster.rb +13 -11
  36. data/lib/html2rss/auto_source/segmenter/list.rb +13 -5
  37. data/lib/html2rss/auto_source/segmenter/primary_link.rb +29 -4
  38. data/lib/html2rss/auto_source/segmenter/semantic.rb +17 -6
  39. data/lib/html2rss/auto_source/segmenter.rb +4 -8
  40. data/lib/html2rss/auto_source.rb +81 -26
  41. data/lib/html2rss/batch.rb +2 -0
  42. data/lib/html2rss/capture/README.md +5 -2
  43. data/lib/html2rss/capture.rb +44 -32
  44. data/lib/html2rss/channel.rb +30 -12
  45. data/lib/html2rss/cli.rb +22 -11
  46. data/lib/html2rss/config/auto_source_contract.rb +52 -45
  47. data/lib/html2rss/config/issue_mapper.rb +4 -1
  48. data/lib/html2rss/config/request_controls.rb +6 -3
  49. data/lib/html2rss/config/schema.rb +15 -5
  50. data/lib/html2rss/config/selectors_validator.rb +2 -2
  51. data/lib/html2rss/config/validation_issue.rb +4 -1
  52. data/lib/html2rss/config/validator.rb +11 -0
  53. data/lib/html2rss/config.rb +53 -27
  54. data/lib/html2rss/contract_error.rb +9 -0
  55. data/lib/html2rss/defaults.rb +21 -23
  56. data/lib/html2rss/doctor/botasaurus.rb +7 -3
  57. data/lib/html2rss/error.rb +10 -0
  58. data/lib/html2rss/error_kind.rb +39 -0
  59. data/lib/html2rss/feed_builder/item_presentation.rb +22 -9
  60. data/lib/html2rss/feed_builder/json_feed/item.rb +2 -1
  61. data/lib/html2rss/feed_builder/json_feed.rb +2 -2
  62. data/lib/html2rss/feed_builder/rss/stylesheet.rb +3 -3
  63. data/lib/html2rss/feed_builder/rss.rb +49 -6
  64. data/lib/html2rss/feed_builder.rb +2 -1
  65. data/lib/html2rss/feed_pipeline/README.md +1 -1
  66. data/lib/html2rss/feed_pipeline/auto_fallback.rb +13 -31
  67. data/lib/html2rss/feed_pipeline/strategy_plan.rb +3 -3
  68. data/lib/html2rss/feed_pipeline.rb +40 -27
  69. data/lib/html2rss/feed_resolution/README.md +1 -1
  70. data/lib/html2rss/feed_resolution/candidate_generator.rb +3 -4
  71. data/lib/html2rss/feed_resolution/policy.rb +1 -1
  72. data/lib/html2rss/feed_resolution/probe.rb +3 -1
  73. data/lib/html2rss/feed_resolution.rb +11 -12
  74. data/lib/html2rss/feed_result.rb +9 -5
  75. data/lib/html2rss/hash_util.rb +3 -3
  76. data/lib/html2rss/html/article_extractor/category_extractor.rb +1 -1
  77. data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +12 -31
  78. data/lib/html2rss/html/article_extractor/heading_extractor.rb +2 -3
  79. data/lib/html2rss/html/article_extractor/id_generator.rb +6 -9
  80. data/lib/html2rss/html/article_extractor/image_extractor.rb +9 -1
  81. data/lib/html2rss/html/article_extractor.rb +19 -47
  82. data/lib/html2rss/html/article_rules/category.rb +3 -3
  83. data/lib/html2rss/html/article_rules/description.rb +6 -15
  84. data/lib/html2rss/html/article_rules/enclosure.rb +70 -42
  85. data/lib/html2rss/html/article_rules/image.rb +30 -16
  86. data/lib/html2rss/html/feed_link.rb +7 -9
  87. data/lib/html2rss/html/navigator/text_extractor.rb +26 -16
  88. data/lib/html2rss/html/navigator.rb +39 -9
  89. data/lib/html2rss/html/probe.rb +4 -0
  90. data/lib/html2rss/html/rendering/description_builder.rb +46 -32
  91. data/lib/html2rss/html/rendering.rb +2 -9
  92. data/lib/html2rss/html/sst_article_extractor.rb +55 -145
  93. data/lib/html2rss/html/title_pieces/kicker.rb +147 -0
  94. data/lib/html2rss/html/title_pieces/nodes.rb +107 -0
  95. data/lib/html2rss/html/title_pieces/nokogiri_nodes.rb +70 -0
  96. data/lib/html2rss/html/title_pieces/sst_nodes.rb +71 -0
  97. data/lib/html2rss/html/title_pieces.rb +169 -0
  98. data/lib/html2rss/html/visibility.rb +56 -0
  99. data/lib/html2rss/html.rb +2 -0
  100. data/lib/html2rss/link_destination/destination_facts.rb +17 -1
  101. data/lib/html2rss/link_destination/noise_policy.rb +11 -10
  102. data/lib/html2rss/link_destination/path_classifier.rb +100 -9
  103. data/lib/html2rss/link_destination/sponsorship/label_text.rb +82 -0
  104. data/lib/html2rss/link_destination/sponsorship.rb +184 -0
  105. data/lib/html2rss/link_destination/text_classifier.rb +104 -8
  106. data/lib/html2rss/mcp/README.md +13 -1
  107. data/lib/html2rss/mcp/config_argument.rb +5 -2
  108. data/lib/html2rss/mcp/contract.rb +13 -4
  109. data/lib/html2rss/mcp/outcome.rb +14 -12
  110. data/lib/html2rss/mcp/runtime.rb +39 -2
  111. data/lib/html2rss/mcp/server.rb +13 -4
  112. data/lib/html2rss/page_recon/README.md +1 -1
  113. data/lib/html2rss/page_recon/diagnostics.rb +2 -0
  114. data/lib/html2rss/recon.rb +10 -12
  115. data/lib/html2rss/request_service/blocked_surface.rb +2 -1
  116. data/lib/html2rss/request_service/botasaurus_contract.rb +25 -3
  117. data/lib/html2rss/request_service/botasaurus_strategy.rb +42 -10
  118. data/lib/html2rss/request_service/context.rb +15 -14
  119. data/lib/html2rss/request_service/cookie_jar.rb +89 -0
  120. data/lib/html2rss/request_service/cookie_store.rb +79 -0
  121. data/lib/html2rss/request_service/feed_advertisement.rb +72 -0
  122. data/lib/html2rss/request_service/httpx_strategy.rb +7 -3
  123. data/lib/html2rss/request_service/local_file_strategy.rb +2 -2
  124. data/lib/html2rss/request_service/policy.rb +48 -11
  125. data/lib/html2rss/request_service/replay_breaker.rb +56 -0
  126. data/lib/html2rss/request_service/response.rb +30 -13
  127. data/lib/html2rss/request_service/response_cache.rb +109 -0
  128. data/lib/html2rss/request_service/response_caching.rb +95 -0
  129. data/lib/html2rss/request_service/strategy.rb +5 -11
  130. data/lib/html2rss/request_service.rb +71 -12
  131. data/lib/html2rss/request_session/pager/base.rb +0 -7
  132. data/lib/html2rss/request_session/pager.rb +1 -1
  133. data/lib/html2rss/request_session.rb +6 -2
  134. data/lib/html2rss/scoring/cluster_scorer.rb +1 -1
  135. data/lib/html2rss/scoring/container_assessor.rb +12 -13
  136. data/lib/html2rss/scoring/engine.rb +6 -6
  137. data/lib/html2rss/scoring/observation.rb +2 -2
  138. data/lib/html2rss/selector_candidates.rb +19 -8
  139. data/lib/html2rss/selectors/README.md +1 -1
  140. data/lib/html2rss/selectors/option_spec.rb +4 -1
  141. data/lib/html2rss/selectors/post_processors/base.rb +0 -8
  142. data/lib/html2rss/selectors/post_processors/gsub.rb +7 -7
  143. data/lib/html2rss/selectors/post_processors/html_transformers/keep_playable_media.rb +50 -0
  144. data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +1 -3
  145. data/lib/html2rss/selectors/post_processors/parse_time.rb +3 -3
  146. data/lib/html2rss/selectors/post_processors/parse_uri.rb +1 -1
  147. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +18 -5
  148. data/lib/html2rss/selectors/post_processors/substring.rb +1 -1
  149. data/lib/html2rss/selectors.rb +53 -12
  150. data/lib/html2rss/sst/attrs.rb +2 -2
  151. data/lib/html2rss/sst/document.rb +3 -3
  152. data/lib/html2rss/sst/index.rb +30 -11
  153. data/lib/html2rss/sst/node.rb +33 -26
  154. data/lib/html2rss/sst/normalizer.rb +31 -19
  155. data/lib/html2rss/sst/tags.rb +113 -2
  156. data/lib/html2rss/sst/text.rb +69 -29
  157. data/lib/html2rss/sst.rb +2 -0
  158. data/lib/html2rss/status.rb +6 -3
  159. data/lib/html2rss/surface_category.rb +1 -1
  160. data/lib/html2rss/syndication/README.md +12 -6
  161. data/lib/html2rss/syndication/alternate_links.rb +1 -1
  162. data/lib/html2rss/syndication/channel_facts.rb +87 -0
  163. data/lib/html2rss/syndication/discovery.rb +82 -28
  164. data/lib/html2rss/syndication/feed.rb +20 -0
  165. data/lib/html2rss/syndication/item_media.rb +103 -0
  166. data/lib/html2rss/syndication/parser.rb +75 -51
  167. data/lib/html2rss/test/enhance_audit.rb +2 -2
  168. data/lib/html2rss/test/item_sample.rb +43 -0
  169. data/lib/html2rss/test.rb +42 -14
  170. data/lib/html2rss/url.rb +115 -14
  171. data/lib/html2rss/version.rb +1 -1
  172. data/lib/html2rss/word_count.rb +53 -0
  173. data/lib/html2rss.rb +17 -8
  174. data/schema/html2rss-config.schema.json +56 -41
  175. metadata +31 -4
  176. data/lib/html2rss/html/rendering/media_renderer.rb +0 -39
  177. data/lib/html2rss/html/rendering/pdf_renderer.rb +0 -30
@@ -103,7 +103,8 @@ module Html2rss
103
103
  {
104
104
  url: enclosure.url.to_s,
105
105
  mime_type: enclosure.type,
106
- size_in_bytes: size&.positive? ? size : nil
106
+ size_in_bytes: (size if size.positive?),
107
+ duration_in_seconds: enclosure.duration
107
108
  }.compact
108
109
  end
109
110
  end
@@ -16,7 +16,7 @@ module Html2rss
16
16
  # @param user_comment [String] required generator comment (from {Status})
17
17
  # @param feed_url [String, nil] optional absolute self URL for the feed (JSON Feed feed_url)
18
18
  def initialize(channel:, articles:, user_comment:, feed_url: nil)
19
- raise ArgumentError, 'user_comment must be a non-blank String' if blank_string?(user_comment)
19
+ raise InvalidOption, 'user_comment must be a non-blank String' if blank_string?(user_comment)
20
20
 
21
21
  @channel = channel
22
22
  @articles = articles
@@ -42,7 +42,7 @@ module Html2rss
42
42
 
43
43
  def normalize_feed_url(value)
44
44
  return if value.nil?
45
- raise ArgumentError, 'feed_url must be a String or nil' unless value.is_a?(String)
45
+ raise InvalidOption, 'feed_url must be a String or nil' unless value.is_a?(String)
46
46
  return if value.strip.empty?
47
47
 
48
48
  Url.from_absolute(value).to_s
@@ -43,9 +43,9 @@ module Html2rss
43
43
  # @param type [String] MIME type (`text/css` or `text/xsl`)
44
44
  # @param media [String] media query hint for the stylesheet
45
45
  def initialize(href:, type:, media: 'all')
46
- raise ArgumentError, 'stylesheet.href must be a String' unless href.is_a?(String)
47
- raise ArgumentError, 'stylesheet.type invalid' unless TYPES.include?(type)
48
- raise ArgumentError, 'stylesheet.media must be a String' unless media.is_a?(String)
46
+ raise InvalidOption, 'stylesheet.href must be a String' unless href.is_a?(String)
47
+ raise InvalidOption, 'stylesheet.type invalid' unless TYPES.include?(type)
48
+ raise InvalidOption, 'stylesheet.media must be a String' unless media.is_a?(String)
49
49
 
50
50
  @href = href
51
51
  @type = type
@@ -6,7 +6,20 @@ module Html2rss
6
6
  module FeedBuilder
7
7
  ##
8
8
  # Builds an RSS Feed by providing channel, articles and stylesheets.
9
- class Rss
9
+ #
10
+ # A media feed ({ItemPresentation.media_feed?}) also gets the podcast tags clients play from:
11
+ # channel +itunes:image+, +itunes:author+, +itunes:category+ (nesting its subcategories) and
12
+ # +itunes:explicit+ (+true+ or +false+), item +itunes:duration+ (+HH:MM:SS+) and +itunes:image+, each
13
+ # only when known; categories and the explicit flag only when the followed native feed states them.
14
+ #
15
+ # +managingEditor+ carries the channel author only when it is an RSS 2.0 email address
16
+ # (+editor@example.com+ or +editor@example.com (Name)+); a plain name is no managing editor.
17
+ class Rss # rubocop:disable Metrics/ClassLength -- channel, item and podcast projections stay co-located
18
+ # First duration in seconds that +itunes:duration+ cannot spell as +HH:MM:SS+ (100 hours).
19
+ MAX_CLOCK_DURATION = 100 * 3600
20
+ # An RSS 2.0 person: an email address, optionally followed by the name in parentheses.
21
+ EMAIL_PERSON = /\A[^\s@()]+@[^\s@()]+(?: \(.+\))?\z/
22
+
10
23
  class << self
11
24
  # @param article [Html2rss::Article] source article
12
25
  # @param item_maker [RSS::Maker::RSS20::ItemsBase::ItemBase] RSS item builder
@@ -60,12 +73,13 @@ module Html2rss
60
73
  # @param stylesheets [Array<Hash>] An optional array of stylesheet configurations.
61
74
  # @param generator [String] required preformatted generator comment (from {Status})
62
75
  def initialize(channel:, articles:, generator:, stylesheets: [])
63
- raise ArgumentError, 'generator must be a non-blank String' if blank_string?(generator)
76
+ raise InvalidOption, 'generator must be a non-blank String' if blank_string?(generator)
64
77
 
65
78
  @channel = channel
66
79
  @articles = articles
67
80
  @stylesheets = stylesheets
68
81
  @generator = generator
82
+ @media_feed = ItemPresentation.media_feed?(articles)
69
83
  end
70
84
 
71
85
  # @return [RSS::Rss] RSS 2.0 document instance
@@ -80,7 +94,7 @@ module Html2rss
80
94
 
81
95
  private
82
96
 
83
- attr_reader :channel, :articles, :generator
97
+ attr_reader :channel, :articles, :generator, :media_feed
84
98
 
85
99
  def blank_string?(value)
86
100
  !value.is_a?(String) || value.strip.empty?
@@ -98,13 +112,25 @@ module Html2rss
98
112
  channel_maker.description = channel.description
99
113
  channel_maker.ttl = channel.ttl
100
114
 
101
- channel_maker.managingEditor = channel.author if channel.author
102
- channel_maker.author = channel.author if channel.author
115
+ channel_maker.managingEditor = channel.author if EMAIL_PERSON.match?(channel.author)
103
116
  channel_maker.link = channel.url.to_s
104
117
  channel_maker.generator = generator
105
118
  channel_maker.updated = channel.last_build_date
106
119
 
107
120
  make_image(maker.image) if channel.image
121
+ make_podcast_channel(channel_maker) if media_feed
122
+ end
123
+
124
+ def make_podcast_channel(channel_maker)
125
+ channel_maker.itunes_image = channel.image
126
+ channel_maker.itunes_author = channel.author
127
+ channel_maker.itunes_explicit = channel.explicit&.to_s
128
+ channel.categories.each { add_itunes_category(channel_maker, *_1) }
129
+ end
130
+
131
+ def add_itunes_category(channel_maker, name, *subcategories)
132
+ category = channel_maker.new_itunes_category(name)
133
+ subcategories.each { category.new_category.text = _1 }
108
134
  end
109
135
 
110
136
  def make_image(image_maker)
@@ -114,9 +140,26 @@ module Html2rss
114
140
 
115
141
  def make_items(maker)
116
142
  articles.each do |article|
117
- maker.items.new_item { |item_maker| self.class.add_item(article, item_maker) }
143
+ maker.items.new_item do |item_maker|
144
+ self.class.add_item(article, item_maker)
145
+ make_podcast_item(article, item_maker) if media_feed
146
+ end
118
147
  end
119
148
  end
149
+
150
+ def make_podcast_item(article, item_maker)
151
+ duration = ItemPresentation.rss_enclosure_for(article)&.duration
152
+ item_maker.itunes_duration = format_duration(duration) if duration
153
+ item_maker.itunes_image = article.image
154
+ end
155
+
156
+ # +HH:MM:SS+; plain seconds from 100 hours on, which +HH+ cannot spell.
157
+ def format_duration(seconds)
158
+ return seconds.to_s if seconds >= MAX_CLOCK_DURATION
159
+
160
+ format('%<hours>02d:%<minutes>02d:%<seconds>02d',
161
+ hours: seconds / 3600, minutes: seconds % 3600 / 60, seconds: seconds % 60)
162
+ end
120
163
  end
121
164
  end
122
165
  end
@@ -8,7 +8,8 @@ module Html2rss
8
8
  # FeedResult; +feed_url+ is a render-time JSON Feed-only option.
9
9
  #
10
10
  # Channel field projection:
11
- # - Rss: language, title, description, ttl, link, updated
11
+ # - Rss: language, title, description, ttl, link, updated, managingEditor (author in email form only)
12
+ # (+ itunes:image, itunes:author, itunes:category, itunes:explicit on a media feed)
12
13
  # - JsonFeed: title, home_page_url, description, language, icon, authors (+ feed_url)
13
14
  module FeedBuilder
14
15
  end
@@ -11,7 +11,7 @@ Use `:auto` when you want `default` (HTTPX) first and a browser-backed hop only
11
11
  1. **Default (HTTPX)** — plain HTTP (faster, cheaper).
12
12
  2. **Botasaurus** — attempted when default raises a fallback-eligible error (for example `BlockedSurfaceDetected` or timeout) or extracts zero feed items.
13
13
 
14
- Before escalating default → Botasaurus on a **weak** auto-source extract (empty / below floor / high-entropy / app-shell / unsupported — not blocked), `FeedResolution` may probe up to five same-origin listing or native-feed candidates and rewrite the effective scrape URL via {ScrapeTarget}. Botasaurus then uses the resolved URL. Direct RSS/Atom entry URLs are parsed via `Syndication::Parser` without HTML AutoSource.
14
+ Before escalating default → Botasaurus on a **weak** auto-source extract (empty / below floor / high-entropy / app-shell / unsupported — not blocked), `FeedResolution` may probe up to five same-origin listing or native-feed candidates and rewrite the effective scrape URL via {ScrapeTarget}. Botasaurus then uses the resolved URL. Direct RSS/Atom/RDF entry URLs are parsed via `Syndication::Parser` without HTML AutoSource.
15
15
 
16
16
  There is no Browserless / Puppeteer-in-gem tier. Pin `botasaurus` when you want browser rendering without default first. Botasaurus needs `BOTASAURUS_SCRAPER_URL`.
17
17
 
@@ -108,26 +108,15 @@ module Html2rss
108
108
  end
109
109
 
110
110
  # @param response [RequestService::Response] successful response
111
- # @param articles [Array] extracted articles
112
- # @param dedup_dropped [Integer] articles removed by deduplication
111
+ # @param extraction [Html2rss::FeedPipeline::Extraction] its articles, drops and feed channel facts
113
112
  # @param selected_strategy [Symbol] concrete strategy that produced items
114
113
  # @param attempt_count [Integer] number of attempts recorded for this chain
115
- # @param admission_drops [Hash{String => Integer}] Cleanup drop tallies
116
114
  # @param scrape_target [Html2rss::ScrapeTarget]
117
115
  # @return [void]
118
- # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- PipelineOutcome kwargs stay co-located
119
- def succeed!(response:, articles:, dedup_dropped:, selected_strategy:, attempt_count:,
120
- scrape_target:, admission_drops: {})
116
+ def succeed!(response:, extraction:, selected_strategy:, attempt_count:, scrape_target:)
121
117
  @result = PipelineOutcome.new(
122
- response:,
123
- articles:,
124
- dedup_dropped:,
125
- selected_strategy:,
126
- attempt_count:,
127
- strategy_attempts: attempts,
128
- admission_drops:,
129
- scrape_target:,
130
- entry_resolution:
118
+ response:, selected_strategy:, attempt_count:, strategy_attempts: attempts,
119
+ scrape_target:, entry_resolution:, **extraction.to_h
131
120
  )
132
121
  end
133
122
 
@@ -215,6 +204,8 @@ module Html2rss
215
204
  rescue *NON_FALLBACK_ERRORS => error
216
205
  state.record_error(strategy:, error:, elapsed_ms: monotonic_ms - started_ms)
217
206
  state.abort!
207
+ rescue Html2rss::ContractError
208
+ raise
218
209
  rescue Html2rss::Error => error
219
210
  state.record_error(strategy:, error:, elapsed_ms: monotonic_ms - started_ms)
220
211
  log_fallback_error(strategy:, next_strategy:, error:, request_session:) if next_strategy && request_session
@@ -222,11 +213,9 @@ module Html2rss
222
213
 
223
214
  # rubocop:disable-next Metrics/MethodLength, Metrics/AbcSize, Metrics/ParameterLists -- extract + resolution delegate
224
215
  def process_response(response:, strategy:, next_strategy:, request_session:, state:, elapsed_ms: nil)
225
- articles, dedup_dropped, admission_drops = pipeline.deduplicated_articles(
226
- config:, response:, request_session:
227
- )
216
+ extraction = pipeline.extract(config:, response:, request_session:)
228
217
  state.remember_response(response)
229
- items_count = articles.size
218
+ items_count = extraction.articles.size
230
219
  state.record_items(strategy:, items_count:, transport_meta: response.transport_meta, elapsed_ms:)
231
220
  Log.debug("#{self.class}: strategy=#{strategy} items=#{items_count} " \
232
221
  "host=#{response.url.host} elapsed=#{format('%.3f', budget.elapsed_seconds)}s " \
@@ -234,16 +223,13 @@ module Html2rss
234
223
 
235
224
  if (outcome = FeedResolution.try_apply!(
236
225
  pipeline:, config:, response:, session: request_session, strategy:, resources:,
237
- articles:, scrape_target:, state:, budget:
226
+ articles: extraction.articles, scrape_target:, state:, budget:
238
227
  ))
239
228
  @scrape_target = outcome.scrape_target
240
229
  return
241
230
  end
242
231
 
243
- if items_count.positive?
244
- return record_success(response:, strategy:, articles:, dedup_dropped:, admission_drops:,
245
- state:)
246
- end
232
+ return record_success(response:, strategy:, extraction:, state:) if items_count.positive?
247
233
 
248
234
  if DETERMINISTIC_HTTP_STATUSES.member?(response&.status)
249
235
  state.abort!
@@ -254,14 +240,10 @@ module Html2rss
254
240
  log_info_fallback_zero_items(strategy:, next_strategy:, response:) if next_strategy
255
241
  end
256
242
 
257
- # rubocop:disable-next Metrics/ParameterLists, Metrics/MethodLength -- success kwargs + structured selected log
258
- def record_success(response:, strategy:, articles:, dedup_dropped:, admission_drops:, state:)
243
+ # rubocop:disable-next Metrics/MethodLength -- success + structured selected log
244
+ def record_success(response:, strategy:, extraction:, state:)
259
245
  attempt_count = state.attempts.size
260
- state.succeed!(
261
- response:, articles:, dedup_dropped:, selected_strategy: strategy,
262
- attempt_count:, admission_drops:,
263
- scrape_target:
264
- )
246
+ state.succeed!(response:, extraction:, selected_strategy: strategy, attempt_count:, scrape_target:)
265
247
  return unless attempt_count > 1
266
248
 
267
249
  Log.info({
@@ -33,13 +33,13 @@ module Html2rss
33
33
  ##
34
34
  # @param name [Symbol, String] config or configure strategy value
35
35
  # @return [Auto, Concrete] resolved feed-level plan
36
- # @raise [ArgumentError] when +name+ is neither +:auto+ nor a registered strategy
36
+ # @raise [InvalidOption] when +name+ is neither +:auto+ nor a registered strategy
37
37
  def resolve(name)
38
38
  normalized = name.to_sym
39
39
  return Auto.new if normalized == AUTO_NAME
40
40
 
41
41
  unless RequestService.strategy_registered?(normalized)
42
- raise ArgumentError, "unknown strategy plan: #{name.inspect}"
42
+ raise InvalidOption, "unknown strategy plan: #{name.inspect}"
43
43
  end
44
44
 
45
45
  Concrete.new(strategy: normalized)
@@ -64,7 +64,7 @@ module Html2rss
64
64
 
65
65
  resolve(name)
66
66
  true
67
- rescue ArgumentError
67
+ rescue InvalidOption
68
68
  false
69
69
  end
70
70
 
@@ -10,11 +10,19 @@ module Html2rss
10
10
  # selected_strategy: set on :auto success; nil otherwise.
11
11
  # attempt_count: auto attempts attempted; 0 outside :auto.
12
12
  # strategy_attempts: auto attempt hashes (with optional transport_meta); empty outside :auto.
13
+ # feed_channel: channel facts of the native feed followed ({Syndication::ChannelFacts}); nil without one.
13
14
  PipelineOutcome = Data.define(
14
15
  :response, :articles, :dedup_dropped, :selected_strategy, :attempt_count, :strategy_attempts,
15
- :admission_drops, :scrape_target, :entry_resolution
16
+ :admission_drops, :scrape_target, :entry_resolution, :feed_channel
16
17
  )
17
18
 
19
+ # What one response yields after extraction and dedup ({#extract}); its fields are the
20
+ # {PipelineOutcome} fields of the same names.
21
+ # articles: unique articles. dedup_dropped: articles removed by deduplication.
22
+ # admission_drops: Cleanup drop tallies. feed_channel: the followed native feed's channel facts
23
+ # ({Syndication::ChannelFacts}); nil without one.
24
+ Extraction = Data.define(:articles, :dedup_dropped, :admission_drops, :feed_channel)
25
+
18
26
  ##
19
27
  # @param raw_config [Hash{Symbol => Object}] user-provided feed config
20
28
  def initialize(raw_config)
@@ -71,11 +79,11 @@ module Html2rss
71
79
  # @param config [Html2rss::Config]
72
80
  # @param response [Html2rss::RequestService::Response]
73
81
  # @param request_session [Html2rss::RequestSession]
74
- # @return [Array(Array<Html2rss::Article>, Integer, Hash)] unique articles, dedup drops, admission drops
75
- def deduplicated_articles(config:, response:, request_session:)
76
- collected, admission_drops = collect_articles(config:, response:, request_session:)
82
+ # @return [Extraction] unique articles with their dedup and admission drops and feed channel facts
83
+ def extract(config:, response:, request_session:)
84
+ collected, admission_drops, feed_channel = collect_articles(config:, response:, request_session:)
77
85
  unique = Article::Deduplicator.new(collected).call
78
- [unique, collected.size - unique.size, admission_drops]
86
+ Extraction.new(articles: unique, dedup_dropped: collected.size - unique.size, admission_drops:, feed_channel:)
79
87
  end
80
88
 
81
89
  private
@@ -95,13 +103,12 @@ module Html2rss
95
103
  def run_pipeline_for_strategy(config, strategy:, resources:)
96
104
  request_session = request_session_for(config, strategy:, resources:)
97
105
  response = request_session.fetch_initial_response
98
- articles, dedup_dropped, admission_drops = deduplicated_articles(config:, response:, request_session:)
99
- raise_empty_auto_source!(strategy:, response:) if config.auto_source && articles.empty?
106
+ extraction = extract(config:, response:, request_session:)
107
+ raise_empty_auto_source!(strategy:, response:) if config.auto_source && extraction.articles.empty?
100
108
 
101
109
  PipelineOutcome.new(
102
- response:, articles:, dedup_dropped:, selected_strategy: nil, attempt_count: 0,
103
- strategy_attempts: [], admission_drops:,
104
- scrape_target: ScrapeTarget.from_config(config), entry_resolution: nil
110
+ response:, selected_strategy: nil, attempt_count: 0, strategy_attempts: [],
111
+ scrape_target: ScrapeTarget.from_config(config), entry_resolution: nil, **extraction.to_h
105
112
  )
106
113
  end
107
114
 
@@ -127,49 +134,55 @@ module Html2rss
127
134
  end
128
135
 
129
136
  def collect_articles(config:, response:, request_session:)
130
- selector = selector_articles(config:, response:, request_session:)
131
- auto, admission_drops = auto_source_articles(config:, response:, request_session:)
132
- [selector + auto, admission_drops]
137
+ selector, selector_drops = selector_articles(config:, response:, request_session:)
138
+ auto, auto_drops, feed_channel = auto_source_articles(config:, response:, request_session:)
139
+ [selector + auto, selector_drops.merge(auto_drops) { |_reason, left, right| left + right }, feed_channel]
133
140
  end
134
141
 
135
- # rubocop:disable-next Metrics/MethodLength
142
+ # @return [Array(Array<Html2rss::Article>, Hash{String => Integer})] articles and unlinked-item drop tallies
136
143
  def selector_articles(config:, response:, request_session:)
137
- return [] unless (selectors = config.selectors)
138
- return [] if response.feed_response?
144
+ return [[], {}] unless (selectors = config.selectors)
145
+ return [[], {}] if response.feed_response?
139
146
 
140
147
  page_responses = request_session.page_responses(
141
148
  response,
142
149
  pagination_config: selectors.dig(:items, :pagination)
143
150
  )
151
+ engines = page_responses.lazy.map { |page| Selectors.new(page, selectors:, time_zone: config.time_zone) }
152
+ collect_selector_pages(engines)
153
+ end
144
154
 
155
+ # Stops at the first empty page after articles were found; drop tallies sum over the pages read.
156
+ def collect_selector_pages(engines)
145
157
  articles = []
146
- page_responses.each do |page_response|
147
- page_articles = Selectors.new(page_response, selectors:, time_zone: config.time_zone).articles
158
+ drops = Hash.new(0)
159
+ engines.each do |engine|
160
+ page_articles = engine.articles
161
+ engine.admission_drops.each { |reason, count| drops[reason] += count }
148
162
  break if page_articles.empty? && articles.any?
149
163
 
150
164
  articles.concat(page_articles)
151
165
  end
152
- articles
166
+ [articles, drops]
153
167
  end
154
168
 
155
- # @return [Array(Array<Html2rss::Article>, Hash{String => Integer})]
169
+ # @return [Array(Array<Html2rss::Article>, Hash{String => Integer}, Html2rss::Syndication::ChannelFacts)]
156
170
  def auto_source_articles(config:, response:, request_session:)
157
- return [[], {}] unless (auto_source = config.auto_source)
171
+ return [[], {}, nil] unless (auto_source = config.auto_source)
158
172
  return syndication_articles(response) if response.feed_response?
159
173
 
160
174
  source = AutoSource.new(response, auto_source, request_session:)
161
- [source.articles, source.admission_drops]
175
+ [source.articles, source.admission_drops, source.feed_channel]
162
176
  end
163
177
 
164
178
  def syndication_articles(response)
165
- articles = Syndication::Parser.parse_response(response).map do |hash|
166
- Article.new(**hash, scraper: AutoSource::Scraper::NativeFeed)
167
- end
168
- [articles, {}]
179
+ feed = Syndication::Parser.parse_response(response)
180
+ articles = feed.articles.map { |hash| Article.new(**hash, scraper: AutoSource::Scraper::NativeFeed) }
181
+ [articles, {}, feed.channel]
169
182
  end
170
183
 
171
184
  def feed_result_for(config, outcome) # rubocop:disable Metrics/MethodLength
172
- channel = Channel.from_response(outcome.response, overrides: config.channel)
185
+ channel = Channel.from_response(outcome.response, overrides: config.channel, feed: outcome.feed_channel)
173
186
  status = Status.build(
174
187
  articles: outcome.articles,
175
188
  dedup_dropped: outcome.dedup_dropped,
@@ -24,7 +24,7 @@ tournament on item count (feed-as-means) — intentional.
24
24
 
25
25
  ### CandidateGenerator mix
26
26
 
27
- - **Feed slot:** first `Syndication::Discovery.candidate_urls` only (alternates precede path guesses).
27
+ - **Feed slot:** first same-domain `Syndication::Discovery.candidate_urls` entry. Alternates precede path guesses, except audio-hinted ones on a page without podcast signals. An alternate on another domain is skipped.
28
28
  - **Listing fill** (cap = `max - 1` when `max > 1`; `max == 1` → feed only): nav (`taxonomy_path +3`, `content_path +1`) → segment first-wins (`:list` → `:cluster` → `:semantic`, ≥2 same-origin primary links) → `LISTING_PATHS`.
29
29
  - Same-page / same-registrable-domain filters; feed never pads into listing slots.
30
30
 
@@ -46,13 +46,12 @@ module Html2rss
46
46
 
47
47
  def listing_cap = max > 1 ? max - 1 : 0
48
48
 
49
+ # First eligible Discovery candidate: an alternate on another domain (Discovery ranks
50
+ # audio-hinted ones first on a podcast page) must not displace a same-domain feed or path guess.
49
51
  def feed_slot
50
52
  return unless parsed_html
51
53
 
52
- url = Syndication::Discovery.candidate_urls(page_url: entry_url, parsed_body: parsed_html).first
53
- return unless eligible_url?(url)
54
-
55
- url
54
+ Syndication::Discovery.candidate_urls(page_url: entry_url, parsed_body: parsed_html).find { eligible_url?(_1) }
56
55
  end
57
56
 
58
57
  def listing_urls
@@ -15,7 +15,7 @@ module Html2rss
15
15
  # @param surface_category [Html2rss::SurfaceCategory, Symbol]
16
16
  # @return [Boolean]
17
17
  def resolve?(config:, articles:, surface_category:)
18
- raise ArgumentError, 'articles must be an Array' unless articles.is_a?(Array)
18
+ raise InvalidError, 'articles must be an Array' unless articles.is_a?(Array)
19
19
  return false unless eligible_config?(config)
20
20
 
21
21
  category = SurfaceCategory[surface_category]
@@ -29,6 +29,8 @@ module Html2rss
29
29
  origin_url:
30
30
  )
31
31
  score_response(response)
32
+ rescue Html2rss::ContractError
33
+ raise
32
34
  rescue Html2rss::Error
33
35
  nil
34
36
  end
@@ -39,7 +41,7 @@ module Html2rss
39
41
 
40
42
  def score_response(response) # rubocop:disable Metrics/MethodLength -- feed vs HTML branches
41
43
  if response.feed_response?
42
- count = Syndication::Parser.parse_response(response).size
44
+ count = Syndication::Parser.parse_response(response).articles.size
43
45
  return Scored.new(
44
46
  url: response.url,
45
47
  score: Scorer.score_feed(articles_count: count),
@@ -8,6 +8,9 @@ module Html2rss
8
8
  #
9
9
  # {include:file:lib/html2rss/feed_resolution/README.md}
10
10
  module FeedResolution
11
+ # Raised when the tournament or its policy is handed articles that are not an Array.
12
+ class InvalidError < Html2rss::ContractError; end
13
+
11
14
  # Default entry-resolution options merged into AutoSource::DEFAULT_CONFIG.
12
15
  DEFAULT_CONFIG = {
13
16
  enabled: true,
@@ -120,7 +123,7 @@ module Html2rss
120
123
  rescue RequestService::RequestBudgetExceeded
121
124
  log_budget_exhausted
122
125
  nil
123
- rescue *NON_FALLBACK_ERRORS
126
+ rescue *NON_FALLBACK_ERRORS, Html2rss::ContractError
124
127
  raise
125
128
  rescue Html2rss::Error => error
126
129
  log_retry_failed(error)
@@ -185,19 +188,15 @@ module Html2rss
185
188
  config, strategy:, resources:, scrape_url: effective.effective_url
186
189
  )
187
190
  retry_response = retry_session.fetch_initial_response
188
- retry_articles, dedup_dropped, admission_drops = pipeline.deduplicated_articles(
189
- config:, response: retry_response, request_session: retry_session
190
- )
191
+ extraction = pipeline.extract(config:, response: retry_response, request_session: retry_session)
192
+ items_count = extraction.articles.size
191
193
  state.remember_response(retry_response)
192
- state.record_items(
193
- strategy:, items_count: retry_articles.size, transport_meta: retry_response.transport_meta
194
- )
195
- return unless retry_articles.size.positive?
194
+ state.record_items(strategy:, items_count:, transport_meta: retry_response.transport_meta)
195
+ return unless items_count.positive?
196
196
 
197
197
  state.succeed!(
198
- response: retry_response, articles: retry_articles, dedup_dropped:,
199
- selected_strategy: strategy, attempt_count: state.attempts.size, admission_drops:,
200
- scrape_target: effective
198
+ response: retry_response, extraction:, selected_strategy: strategy,
199
+ attempt_count: state.attempts.size, scrape_target: effective
201
200
  )
202
201
  true
203
202
  end
@@ -214,7 +213,7 @@ module Html2rss
214
213
  # @param surface_category [Symbol, nil]
215
214
  # rubocop:disable-next Metrics/ParameterLists -- tournament context stays co-located
216
215
  def initialize(entry_url:, response:, session:, config:, articles:, surface_category: nil)
217
- raise ArgumentError, 'articles must be an Array' unless articles.is_a?(Array)
216
+ raise InvalidError, 'articles must be an Array' unless articles.is_a?(Array)
218
217
 
219
218
  @entry_url = Url.from_absolute(entry_url)
220
219
  @response = response
@@ -15,16 +15,20 @@ module Html2rss
15
15
  # are re-frozen via private {#marshal_load}. For trusted cache reads only,
16
16
  # +Marshal.load(payload, freeze: true)+ is also safe.
17
17
  class FeedResult
18
+ # Raised when the scrape handle is built from invalid parts.
19
+ class InvalidError < Html2rss::ContractError; end
20
+
18
21
  ##
19
22
  # @param channel [Html2rss::Channel] channel metadata (internal; not exposed on the public API)
20
23
  # @param articles [Array<Html2rss::Article>] extracted articles (deduplicated; not exposed)
21
24
  # @param status [Html2rss::Status] pipeline telemetry
22
25
  # @param stylesheets [Array<Hash>] optional RSS stylesheet configs
26
+ # @raise [InvalidError] when a part has the wrong type
23
27
  def initialize(channel:, articles:, status:, stylesheets: [])
24
- raise ArgumentError, 'channel must be a Html2rss::Channel' unless channel.is_a?(Channel)
25
- raise ArgumentError, 'status must be a Html2rss::Status' unless status.is_a?(Status)
26
- raise ArgumentError, 'articles must be an Array' unless articles.is_a?(Array)
27
- raise ArgumentError, 'articles must all be Html2rss::Article' unless articles.all?(Article)
28
+ raise InvalidError, 'channel must be a Html2rss::Channel' unless channel.is_a?(Channel)
29
+ raise InvalidError, 'status must be a Html2rss::Status' unless status.is_a?(Status)
30
+ raise InvalidError, 'articles must be an Array' unless articles.is_a?(Array)
31
+ raise InvalidError, 'articles must all be Html2rss::Article' unless articles.all?(Article)
28
32
 
29
33
  @channel = channel
30
34
  @articles = articles.dup.freeze
@@ -81,7 +85,7 @@ module Html2rss
81
85
  end
82
86
 
83
87
  def freeze_stylesheets(stylesheets)
84
- raise ArgumentError, 'stylesheets must be an Array' unless stylesheets.is_a?(Array)
88
+ raise InvalidError, 'stylesheets must be an Array' unless stylesheets.is_a?(Array)
85
89
 
86
90
  stylesheets.map do |sheet|
87
91
  sheet.to_h.transform_keys(&:to_sym).transform_values do |value|
@@ -81,7 +81,7 @@ module Html2rss
81
81
 
82
82
  unless value.each_key.all?(Symbol)
83
83
  invalid_key = value.keys.find { _1.class != Symbol }
84
- raise ArgumentError, "#{context} must use symbol keys (found #{invalid_key.inspect})"
84
+ raise InvalidOption, "#{context} must use symbol keys (found #{invalid_key.inspect})"
85
85
  end
86
86
 
87
87
  value.each_value { assert_symbol_keys!(_1, context:, deep:) } if deep
@@ -98,7 +98,7 @@ module Html2rss
98
98
 
99
99
  unless value.each_key.all?(String)
100
100
  invalid_key = value.keys.find { _1.class != String }
101
- raise ArgumentError, "#{context} must use string keys (found #{invalid_key.inspect})"
101
+ raise InvalidOption, "#{context} must use string keys (found #{invalid_key.inspect})"
102
102
  end
103
103
 
104
104
  value.each_value { assert_string_keys!(_1, context:, deep:) } if deep
@@ -111,7 +111,7 @@ module Html2rss
111
111
  in Symbol then key
112
112
  in String then key.to_sym
113
113
  else
114
- raise ArgumentError, "#{context} must use string or symbol keys (found #{key.inspect})"
114
+ raise InvalidOption, "#{context} must use string or symbol keys (found #{key.inspect})"
115
115
  end
116
116
  end
117
117
  end
@@ -42,7 +42,7 @@ module Html2rss
42
42
  Set.new.tap do |categories|
43
43
  extract_element_data_categories!(categories, article_tag, title:)
44
44
  article_tag.css(CATEGORY_SELECTORS.join(',')).each do |element|
45
- if ArticleRules::Category.class_match?(element['class'])
45
+ if ArticleRules::Category.class_match?(element['class'].to_s.split)
46
46
  extract_text_categories!(categories, element, title:)
47
47
  end
48
48
  extract_element_data_categories!(categories, element, title:)