html2rss 0.32.4 → 0.34.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +4 -2
  3. data/lib/html2rss/article/deduplicator.rb +5 -2
  4. data/lib/html2rss/article/enclosure.rb +185 -25
  5. data/lib/html2rss/article/title.rb +203 -0
  6. data/lib/html2rss/article.rb +79 -13
  7. data/lib/html2rss/auto_source/README.md +16 -8
  8. data/lib/html2rss/auto_source/cleanup.rb +127 -53
  9. data/lib/html2rss/auto_source/corroboration.rb +167 -0
  10. data/lib/html2rss/auto_source/page_links/anchor_claims.rb +82 -0
  11. data/lib/html2rss/auto_source/page_links/card_units.rb +77 -0
  12. data/lib/html2rss/auto_source/page_links/label_binding.rb +79 -0
  13. data/lib/html2rss/auto_source/page_links/sponsored_cards.rb +147 -0
  14. data/lib/html2rss/auto_source/page_links.rb +164 -0
  15. data/lib/html2rss/auto_source/scraper/html.rb +11 -45
  16. data/lib/html2rss/auto_source/scraper/json_document.rb +3 -6
  17. data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +202 -125
  18. data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +27 -38
  19. data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +123 -123
  20. data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +27 -35
  21. data/lib/html2rss/auto_source/scraper/json_state.rb +4 -9
  22. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +27 -5
  23. data/lib/html2rss/auto_source/scraper/microdata.rb +233 -248
  24. data/lib/html2rss/auto_source/scraper/microformats2.rb +10 -4
  25. data/lib/html2rss/auto_source/scraper/native_feed.rb +28 -18
  26. data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +7 -2
  27. data/lib/html2rss/auto_source/scraper/schema/item_list.rb +5 -3
  28. data/lib/html2rss/auto_source/scraper/schema/thing.rb +140 -22
  29. data/lib/html2rss/auto_source/scraper/schema.rb +52 -8
  30. data/lib/html2rss/auto_source/scraper/semantic_html/entry_deduplicator.rb +3 -9
  31. data/lib/html2rss/auto_source/scraper/semantic_html.rb +12 -20
  32. data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +1 -4
  33. data/lib/html2rss/auto_source/scraper/sitemap.rb +7 -14
  34. data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +2 -2
  35. data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb +1 -3
  36. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +7 -7
  37. data/lib/html2rss/auto_source/scraper/xhr_articles.rb +2 -4
  38. data/lib/html2rss/auto_source/scraper.rb +10 -13
  39. data/lib/html2rss/auto_source/segment.rb +19 -5
  40. data/lib/html2rss/auto_source/segmenter/cluster.rb +18 -22
  41. data/lib/html2rss/auto_source/segmenter/list.rb +35 -59
  42. data/lib/html2rss/auto_source/segmenter/primary_link.rb +29 -4
  43. data/lib/html2rss/auto_source/segmenter/semantic.rb +23 -19
  44. data/lib/html2rss/auto_source/segmenter.rb +4 -8
  45. data/lib/html2rss/auto_source.rb +81 -26
  46. data/lib/html2rss/batch.rb +24 -32
  47. data/lib/html2rss/capture/README.md +6 -3
  48. data/lib/html2rss/capture.rb +51 -59
  49. data/lib/html2rss/channel.rb +30 -12
  50. data/lib/html2rss/cli/render.rb +97 -96
  51. data/lib/html2rss/cli/validate.rb +2 -4
  52. data/lib/html2rss/cli.rb +55 -38
  53. data/lib/html2rss/config/auto_source_contract.rb +52 -45
  54. data/lib/html2rss/config/issue_mapper.rb +4 -1
  55. data/lib/html2rss/config/request_controls.rb +16 -5
  56. data/lib/html2rss/config/schema.rb +23 -15
  57. data/lib/html2rss/config/selectors_validator.rb +3 -2
  58. data/lib/html2rss/config/validation_issue.rb +4 -1
  59. data/lib/html2rss/config/validator.rb +14 -3
  60. data/lib/html2rss/config.rb +54 -28
  61. data/lib/html2rss/contract_error.rb +9 -0
  62. data/lib/html2rss/defaults.rb +25 -38
  63. data/lib/html2rss/doctor/botasaurus.rb +18 -22
  64. data/lib/html2rss/error.rb +17 -3
  65. data/lib/html2rss/error_kind.rb +39 -0
  66. data/lib/html2rss/feed_builder/item_presentation.rb +22 -9
  67. data/lib/html2rss/feed_builder/json_feed/item.rb +2 -1
  68. data/lib/html2rss/feed_builder/json_feed.rb +2 -2
  69. data/lib/html2rss/feed_builder/rss/stylesheet.rb +3 -3
  70. data/lib/html2rss/feed_builder/rss.rb +50 -7
  71. data/lib/html2rss/feed_builder.rb +2 -1
  72. data/lib/html2rss/feed_pipeline/README.md +1 -1
  73. data/lib/html2rss/feed_pipeline/auto_fallback.rb +16 -34
  74. data/lib/html2rss/feed_pipeline/strategy_plan.rb +3 -3
  75. data/lib/html2rss/feed_pipeline.rb +40 -27
  76. data/lib/html2rss/feed_resolution/README.md +1 -1
  77. data/lib/html2rss/feed_resolution/candidate_generator.rb +6 -8
  78. data/lib/html2rss/feed_resolution/options.rb +3 -7
  79. data/lib/html2rss/feed_resolution/policy.rb +29 -33
  80. data/lib/html2rss/feed_resolution/probe.rb +6 -4
  81. data/lib/html2rss/feed_resolution/scorer.rb +29 -29
  82. data/lib/html2rss/feed_resolution.rb +16 -21
  83. data/lib/html2rss/feed_result.rb +11 -6
  84. data/lib/html2rss/hash_util.rb +91 -90
  85. data/lib/html2rss/html/article_extractor/category_extractor.rb +7 -10
  86. data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
  87. data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +12 -33
  88. data/lib/html2rss/html/article_extractor/heading_extractor.rb +2 -3
  89. data/lib/html2rss/html/article_extractor/id_generator.rb +7 -10
  90. data/lib/html2rss/html/article_extractor/image_extractor.rb +9 -1
  91. data/lib/html2rss/html/article_extractor.rb +19 -47
  92. data/lib/html2rss/html/article_rules/category.rb +3 -3
  93. data/lib/html2rss/html/article_rules/description.rb +6 -15
  94. data/lib/html2rss/html/article_rules/enclosure.rb +70 -42
  95. data/lib/html2rss/html/article_rules/image.rb +30 -16
  96. data/lib/html2rss/html/feed_link.rb +7 -9
  97. data/lib/html2rss/html/navigator/text_extractor.rb +26 -16
  98. data/lib/html2rss/html/navigator.rb +39 -9
  99. data/lib/html2rss/html/probe.rb +55 -51
  100. data/lib/html2rss/html/rendering/description_builder.rb +46 -32
  101. data/lib/html2rss/html/rendering.rb +2 -9
  102. data/lib/html2rss/html/sst_article_extractor.rb +55 -145
  103. data/lib/html2rss/html/title_pieces/kicker.rb +147 -0
  104. data/lib/html2rss/html/title_pieces/nodes.rb +107 -0
  105. data/lib/html2rss/html/title_pieces/nokogiri_nodes.rb +70 -0
  106. data/lib/html2rss/html/title_pieces/sst_nodes.rb +71 -0
  107. data/lib/html2rss/html/title_pieces.rb +169 -0
  108. data/lib/html2rss/html/visibility.rb +56 -0
  109. data/lib/html2rss/html.rb +2 -0
  110. data/lib/html2rss/link_destination/destination_facts.rb +17 -1
  111. data/lib/html2rss/link_destination/noise_policy.rb +11 -10
  112. data/lib/html2rss/link_destination/path_classifier.rb +111 -42
  113. data/lib/html2rss/link_destination/sponsorship/label_text.rb +82 -0
  114. data/lib/html2rss/link_destination/sponsorship.rb +184 -0
  115. data/lib/html2rss/link_destination/text_classifier.rb +104 -8
  116. data/lib/html2rss/mcp/README.md +13 -1
  117. data/lib/html2rss/mcp/config_argument.rb +14 -4
  118. data/lib/html2rss/mcp/contract.rb +20 -9
  119. data/lib/html2rss/mcp/outcome.rb +18 -17
  120. data/lib/html2rss/mcp/runtime.rb +53 -23
  121. data/lib/html2rss/mcp/server/tools.rb +9 -9
  122. data/lib/html2rss/mcp/server.rb +13 -4
  123. data/lib/html2rss/option.rb +48 -0
  124. data/lib/html2rss/page_recon/README.md +1 -1
  125. data/lib/html2rss/page_recon/diagnostics.rb +9 -9
  126. data/lib/html2rss/page_recon.rb +0 -4
  127. data/lib/html2rss/recon.rb +19 -23
  128. data/lib/html2rss/request_service/blocked_surface.rb +5 -8
  129. data/lib/html2rss/request_service/botasaurus_contract.rb +25 -3
  130. data/lib/html2rss/request_service/botasaurus_strategy.rb +44 -12
  131. data/lib/html2rss/request_service/budget.rb +3 -24
  132. data/lib/html2rss/request_service/compressed_body.rb +36 -48
  133. data/lib/html2rss/request_service/context.rb +16 -17
  134. data/lib/html2rss/request_service/cookie_jar.rb +89 -0
  135. data/lib/html2rss/request_service/cookie_store.rb +79 -0
  136. data/lib/html2rss/request_service/feed_advertisement.rb +72 -0
  137. data/lib/html2rss/request_service/httpx_strategy.rb +7 -3
  138. data/lib/html2rss/request_service/local_file_strategy.rb +2 -3
  139. data/lib/html2rss/request_service/network_guard.rb +3 -2
  140. data/lib/html2rss/request_service/policy.rb +55 -36
  141. data/lib/html2rss/request_service/replay_breaker.rb +56 -0
  142. data/lib/html2rss/request_service/response.rb +33 -19
  143. data/lib/html2rss/request_service/response_cache.rb +109 -0
  144. data/lib/html2rss/request_service/response_caching.rb +95 -0
  145. data/lib/html2rss/request_service/strategy.rb +5 -11
  146. data/lib/html2rss/request_service.rb +71 -12
  147. data/lib/html2rss/request_session/pager/base.rb +3 -10
  148. data/lib/html2rss/request_session/pager/custom_selector.rb +4 -4
  149. data/lib/html2rss/request_session/pager/offset.rb +1 -7
  150. data/lib/html2rss/request_session/pager/rel_next.rb +1 -1
  151. data/lib/html2rss/request_session/pager/url_template.rb +1 -7
  152. data/lib/html2rss/request_session/pager.rb +1 -1
  153. data/lib/html2rss/request_session.rb +6 -2
  154. data/lib/html2rss/scoring/cluster_scorer.rb +1 -1
  155. data/lib/html2rss/scoring/container_assessor.rb +12 -13
  156. data/lib/html2rss/scoring/engine.rb +10 -38
  157. data/lib/html2rss/scoring/link_resolver.rb +2 -7
  158. data/lib/html2rss/scoring/observation.rb +2 -2
  159. data/lib/html2rss/selector_candidates.rb +19 -8
  160. data/lib/html2rss/selectors/README.md +1 -1
  161. data/lib/html2rss/selectors/extractors/href.rb +2 -4
  162. data/lib/html2rss/selectors/json_xml.rb +33 -34
  163. data/lib/html2rss/selectors/option_spec.rb +4 -1
  164. data/lib/html2rss/selectors/post_processors/base.rb +0 -8
  165. data/lib/html2rss/selectors/post_processors/gsub.rb +9 -10
  166. data/lib/html2rss/selectors/post_processors/html_transformers/keep_playable_media.rb +50 -0
  167. data/lib/html2rss/selectors/post_processors/html_transformers/transform_urls_to_absolute_ones.rb +1 -3
  168. data/lib/html2rss/selectors/post_processors/parse_time.rb +5 -6
  169. data/lib/html2rss/selectors/post_processors/parse_uri.rb +1 -1
  170. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +18 -5
  171. data/lib/html2rss/selectors/post_processors/substring.rb +1 -1
  172. data/lib/html2rss/selectors/schema_export.rb +60 -60
  173. data/lib/html2rss/selectors.rb +67 -30
  174. data/lib/html2rss/sst/attrs.rb +2 -2
  175. data/lib/html2rss/sst/document.rb +3 -3
  176. data/lib/html2rss/sst/index.rb +30 -11
  177. data/lib/html2rss/sst/node.rb +42 -29
  178. data/lib/html2rss/sst/normalizer.rb +47 -34
  179. data/lib/html2rss/sst/tags.rb +113 -2
  180. data/lib/html2rss/sst/text.rb +94 -58
  181. data/lib/html2rss/sst.rb +2 -0
  182. data/lib/html2rss/status.rb +27 -31
  183. data/lib/html2rss/surface_category.rb +1 -1
  184. data/lib/html2rss/syndication/README.md +12 -6
  185. data/lib/html2rss/syndication/alternate_links.rb +39 -0
  186. data/lib/html2rss/syndication/channel_facts.rb +87 -0
  187. data/lib/html2rss/syndication/discovery.rb +171 -187
  188. data/lib/html2rss/syndication/feed.rb +20 -0
  189. data/lib/html2rss/syndication/item_media.rb +103 -0
  190. data/lib/html2rss/syndication/parser.rb +117 -116
  191. data/lib/html2rss/test/enhance_audit.rb +15 -17
  192. data/lib/html2rss/test/item_sample.rb +43 -0
  193. data/lib/html2rss/test/policy.rb +9 -11
  194. data/lib/html2rss/test.rb +66 -57
  195. data/lib/html2rss/url.rb +163 -62
  196. data/lib/html2rss/version.rb +1 -1
  197. data/lib/html2rss/word_count.rb +53 -0
  198. data/lib/html2rss.rb +19 -9
  199. data/schema/html2rss-config.schema.json +56 -41
  200. metadata +33 -4
  201. data/lib/html2rss/html/rendering/media_renderer.rb +0 -39
  202. data/lib/html2rss/html/rendering/pdf_renderer.rb +0 -30
@@ -11,7 +11,7 @@ module Html2rss
11
11
  #
12
12
  # Configuration is validated during initialization.
13
13
  class Config # rubocop:disable Metrics/ClassLength
14
- # Raised when a configuration hash fails runtime validation.
14
+ # Raised when a configuration cannot be loaded (YAML shape, file, feed name) or fails runtime validation.
15
15
  class InvalidConfig < Html2rss::Error; end
16
16
 
17
17
  class << self
@@ -25,7 +25,7 @@ module Html2rss
25
25
  # @return [Array(Hash, Html2rss::Config::ValidationReport)]
26
26
  def resolve_and_validate(config_input, feed_name: nil, params: {})
27
27
  raw = resolve_raw_hash(config_input, feed_name)
28
- rescue Psych::Exception, ArgumentError => error
28
+ rescue InvalidConfig => error
29
29
  [{}, IssueMapper.parse_failure(error.message)]
30
30
  else
31
31
  working = HashUtil.deep_dup(raw)
@@ -66,14 +66,16 @@ module Html2rss
66
66
  #
67
67
  # @param string [String] YAML document
68
68
  # @return [Hash{Symbol => Object}] configuration hash
69
- # @raise [ArgumentError] if +string+ is not a String or does not deserialize to a Hash
69
+ # @raise [InvalidConfig] if +string+ is not a String or does not deserialize to a Hash
70
70
  def from_yaml(string)
71
- raise ArgumentError, 'YAML must be a String' unless string.is_a?(String)
71
+ raise InvalidConfig, 'YAML must be a String' unless string.is_a?(String)
72
72
 
73
73
  parsed = YAML.safe_load(string)
74
- raise ArgumentError, 'YAML must deserialize to a Hash' unless parsed.is_a?(Hash)
74
+ raise InvalidConfig, 'YAML must deserialize to a Hash' unless parsed.is_a?(Hash)
75
75
 
76
- HashUtil.deep_symbolize_keys(parsed, context: 'config')
76
+ symbolize_config(parsed)
77
+ rescue Psych::Exception => error
78
+ raise InvalidConfig, "YAML could not be parsed: #{error.message}"
77
79
  end
78
80
 
79
81
  ##
@@ -85,25 +87,16 @@ module Html2rss
85
87
  # @param feed_name [String, nil] the feed name when using multiple feeds.
86
88
  # @param multiple_feeds_key [Symbol] the key under which multiple feeds are defined.
87
89
  # @return [Hash{Symbol => Object}] the configuration hash.
88
- # @raise [ArgumentError] if the file doesn't exist or feed is not found.
89
- # rubocop:disable-next Metrics/MethodLength
90
+ # @raise [InvalidConfig] if the file doesn't exist or feed is not found.
90
91
  def load_yaml(file, feed_name = nil, multiple_feeds_key: MultipleFeedsConfig::CONFIG_KEY_FEEDS)
91
- raise ArgumentError, "File '#{file}' does not exist" unless File.exist?(file)
92
- raise ArgumentError, "`#{multiple_feeds_key}` is a reserved feed name" if feed_name == multiple_feeds_key
92
+ raise InvalidConfig, "File '#{file}' does not exist" unless File.exist?(file)
93
+ raise InvalidConfig, "`#{multiple_feeds_key}` is a reserved feed name" if feed_name == multiple_feeds_key
93
94
 
94
- yaml = YAML.safe_load_file(file, symbolize_names: true)
95
+ yaml = read_yaml_mapping(file)
95
96
 
96
97
  return yaml unless yaml.key?(multiple_feeds_key)
97
98
 
98
- unless feed_name
99
- available_feeds = yaml.fetch(multiple_feeds_key).keys.join(', ')
100
- raise ArgumentError,
101
- "Feed name is required under `#{multiple_feeds_key}`. Available feeds: #{available_feeds}"
102
- end
103
-
104
- config = yaml.dig(multiple_feeds_key, feed_name.to_sym)
105
- raise ArgumentError, "Feed '#{feed_name}' not found under `#{multiple_feeds_key}` key." unless config
106
-
99
+ config = select_feed(file, feed_name, yaml.fetch(multiple_feeds_key), multiple_feeds_key)
107
100
  MultipleFeedsConfig.to_single_feed(config, yaml, multiple_feeds_key:)
108
101
  end
109
102
 
@@ -126,17 +119,15 @@ module Html2rss
126
119
  # @param request_controls [Html2rss::Config::RequestControls, nil] explicit request controls to write
127
120
  # @param limit [Integer, nil] max articles to keep in the auto-sourced feed
128
121
  # @param keep_different_domain [Boolean, nil] when set, overrides cleanup.keep_different_domain
122
+ # @param keep_sponsored [Boolean, nil] when set, overrides cleanup.keep_sponsored
129
123
  # @return [Hash{Symbol => Object}] feed config hash ready for {from_hash}
130
- # rubocop:disable-next Metrics/MethodLength -- keep_different_domain override stays co-located
124
+ # rubocop:disable-next Metrics/MethodLength, Metrics/ParameterLists -- cleanup overrides stay co-located
131
125
  def auto_source_config(url:, items_selector: nil, request_controls: nil, limit: nil,
132
- keep_different_domain: nil)
126
+ keep_different_domain: nil, keep_sponsored: nil)
133
127
  auto_source = AutoSource::DEFAULT_CONFIG
134
- auto_source = auto_source.merge(limit:) unless limit.nil?
135
- unless keep_different_domain.nil?
136
- auto_source = auto_source.merge(
137
- cleanup: auto_source[:cleanup].merge(keep_different_domain:)
138
- )
139
- end
128
+ auto_source = auto_source.merge(limit: Option.integer(:limit, limit, min: 1)) unless limit.nil?
129
+ cleanup = { keep_different_domain:, keep_sponsored: }.compact
130
+ auto_source = auto_source.merge(cleanup: auto_source[:cleanup].merge(cleanup)) unless cleanup.empty?
140
131
 
141
132
  config = {
142
133
  channel: default_config[:channel].merge(url:),
@@ -228,6 +219,41 @@ module Html2rss
228
219
  end
229
220
  end
230
221
  private :resolve_raw_hash
222
+
223
+ def read_yaml_mapping(file)
224
+ yaml = YAML.safe_load_file(file)
225
+ return symbolize_config(yaml) if yaml.is_a?(Hash)
226
+
227
+ raise InvalidConfig, "File '#{file}' must contain a YAML mapping (a Hash)"
228
+ rescue Psych::Exception => error
229
+ raise InvalidConfig, "File '#{file}' could not be parsed: #{error.message}"
230
+ end
231
+ private :read_yaml_mapping
232
+
233
+ # @return [Hash{Symbol => Object}] the named feed's config from the +feeds+ mapping
234
+ def select_feed(file, feed_name, feeds, key)
235
+ unless feeds.is_a?(Hash)
236
+ raise InvalidConfig, "File '#{file}': `#{key}` must be a mapping of feed names to configs"
237
+ end
238
+ unless feed_name
239
+ raise InvalidConfig, "Feed name is required under `#{key}`. Available feeds: #{feeds.keys.join(', ')}"
240
+ end
241
+
242
+ config = feeds[feed_name.to_sym]
243
+ raise InvalidConfig, "Feed '#{feed_name}' not found under `#{key}` key." unless config
244
+ raise InvalidConfig, "File '#{file}': feed '#{feed_name}' must be a mapping" unless config.is_a?(Hash)
245
+
246
+ config
247
+ end
248
+ private :select_feed
249
+
250
+ # Loaded config keys that cannot be symbolized (non-string, non-symbol) are a config-load failure.
251
+ def symbolize_config(parsed)
252
+ HashUtil.deep_symbolize_keys(parsed, context: 'config')
253
+ rescue InvalidOption => error
254
+ raise InvalidConfig, error.message
255
+ end
256
+ private :symbolize_config
231
257
  end
232
258
 
233
259
  ##
@@ -0,0 +1,9 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Html2rss
4
+ ##
5
+ # Base for programmer errors: a value object built from wrong parts (each owner's +InvalidError+) or a
6
+ # component breaking its contract ({AutoSource::ScraperContractError}). It is an {Error}, but no
7
+ # operational rescue may swallow it: every broad +rescue Html2rss::Error+ in +lib/+ re-raises it first.
8
+ class ContractError < Error; end
9
+ end
@@ -55,9 +55,9 @@ module Html2rss
55
55
  #
56
56
  # @param logger [Logger]
57
57
  # @return [Logger] the logger
58
- # @raise [ArgumentError] if logger is not a Logger
58
+ # @raise [InvalidOption] if logger is not a Logger
59
59
  def logger=(logger)
60
- raise ArgumentError, 'logger must be a Logger' unless logger.is_a?(::Logger)
60
+ raise InvalidOption, 'logger must be a Logger' unless logger.is_a?(Logger)
61
61
 
62
62
  @logger = logger
63
63
  @logger.level = @log_level
@@ -69,7 +69,7 @@ module Html2rss
69
69
  #
70
70
  # @param level [Symbol, String, Integer] the new log level
71
71
  # @return [Integer] the normalized log level
72
- # @raise [ArgumentError] if the log level is invalid
72
+ # @raise [InvalidOption] if the log level is invalid
73
73
  def log_level=(level)
74
74
  @log_level = normalize_log_level(level)
75
75
  @logger.level = @log_level
@@ -80,13 +80,13 @@ module Html2rss
80
80
  #
81
81
  # @param formatter [Proc, Logger::Formatter, nil] the new logger formatter
82
82
  # @return [Proc, Logger::Formatter, nil] the new logger formatter
83
- # @raise [ArgumentError] if formatter is not a Proc, Logger::Formatter, or nil
83
+ # @raise [InvalidOption] if formatter is not a Proc, Logger::Formatter, or nil
84
84
  def logger_formatter=(formatter)
85
85
  @logger_formatter = case formatter
86
86
  when nil, Proc, Logger::Formatter
87
87
  formatter
88
88
  else
89
- raise ArgumentError, 'formatter must be a Proc, Logger::Formatter, or nil'
89
+ raise InvalidOption, 'formatter must be a Proc, Logger::Formatter, or nil'
90
90
  end
91
91
  @logger.formatter = @logger_formatter
92
92
  end
@@ -96,14 +96,14 @@ module Html2rss
96
96
  #
97
97
  # @param headers [Hash, Proc, nil] the HTTP request headers to globally apply
98
98
  # @return [Hash, Proc, nil] the assigned headers
99
- # @raise [ArgumentError] if headers is not a Hash, Proc, or nil
99
+ # @raise [InvalidOption] if headers is not a Hash, Proc, or nil
100
100
  def headers=(headers)
101
101
  @headers = case headers
102
102
  when nil then nil
103
103
  when Hash then headers.dup.freeze
104
104
  when Proc then headers
105
105
  else
106
- raise ArgumentError, 'headers must be a Hash or Proc'
106
+ raise InvalidOption, 'headers must be a Hash or Proc'
107
107
  end
108
108
  end
109
109
 
@@ -114,20 +114,9 @@ module Html2rss
114
114
  #
115
115
  # @param strategy [Symbol, String, nil] the strategy plan name
116
116
  # @return [Symbol, nil] the normalized strategy plan name
117
- # @raise [ArgumentError] if the strategy plan is unknown
117
+ # @raise [InvalidOption] if the strategy plan is unknown
118
118
  def default_strategy=(strategy)
119
- if strategy.nil?
120
- @default_strategy = nil
121
- else
122
- unless strategy.is_a?(Symbol) || strategy.is_a?(String)
123
- raise ArgumentError, 'strategy must be a Symbol or String'
124
- end
125
-
126
- normalized = strategy.to_sym
127
- raise ArgumentError, "unknown strategy: #{strategy}" unless FeedPipeline::StrategyPlan.valid?(normalized)
128
-
129
- @default_strategy = normalized
130
- end
119
+ @default_strategy = (Option.strategy(strategy) unless strategy.nil?)
131
120
  end
132
121
 
133
122
  ##
@@ -135,18 +124,9 @@ module Html2rss
135
124
  #
136
125
  # @param ttl [Integer, String, nil] the minimum TTL
137
126
  # @return [Integer, nil] the normalized minimum TTL
138
- # @raise [ArgumentError] if ttl is not a positive integer
127
+ # @raise [InvalidOption] if ttl is not a positive integer
139
128
  def min_ttl=(ttl)
140
- if ttl.nil?
141
- @min_ttl = nil
142
- else
143
- val = Integer(ttl)
144
- raise ArgumentError unless val.positive?
145
-
146
- @min_ttl = val
147
- end
148
- rescue ArgumentError, TypeError
149
- raise ArgumentError, "min_ttl must be a positive integer, got #{ttl.inspect}"
129
+ @min_ttl = (positive_ttl(ttl) unless ttl.nil?)
150
130
  end
151
131
 
152
132
  ##
@@ -154,10 +134,10 @@ module Html2rss
154
134
  #
155
135
  # @param stylesheets [Array<Hash>] the XML stylesheet processing instructions to include in the generated feed
156
136
  # @return [Array<Hash>] the assigned stylesheets
157
- # @raise [ArgumentError] if stylesheets is not an Array of hashes
137
+ # @raise [InvalidOption] if stylesheets is not an Array of hashes
158
138
  def stylesheets=(stylesheets)
159
- raise ArgumentError, 'stylesheets must be an Array' unless stylesheets.is_a?(Array)
160
- raise ArgumentError, 'stylesheets must be an Array of Hashes' unless stylesheets.all?(Hash)
139
+ raise InvalidOption, 'stylesheets must be an Array' unless stylesheets.is_a?(Array)
140
+ raise InvalidOption, 'stylesheets must be an Array of Hashes' unless stylesheets.all?(Hash)
161
141
 
162
142
  @stylesheets = stylesheets.map { |h| h.dup.freeze }.freeze
163
143
  end
@@ -171,19 +151,26 @@ module Html2rss
171
151
  # @return [void]
172
152
  def initialize_copy(other)
173
153
  super
174
- @headers = @headers.dup if @headers.is_a?(Hash)
175
- @stylesheets = @stylesheets.map(&:dup) if @stylesheets.is_a?(Array)
154
+ @headers = other.headers.dup if other.headers.is_a?(Hash)
155
+ @stylesheets = other.stylesheets.map(&:dup)
176
156
  end
177
157
 
178
158
  private
179
159
 
160
+ def positive_ttl(ttl)
161
+ val = Integer(ttl, exception: false)
162
+ return val if val&.positive?
163
+
164
+ raise InvalidOption, "min_ttl must be a positive integer, got #{ttl.inspect}"
165
+ end
166
+
180
167
  def normalize_log_level(level)
181
168
  if level.is_a?(Integer)
182
- raise ArgumentError, "invalid log level: #{level}" unless LOG_LEVELS.value?(level)
169
+ raise InvalidOption, "invalid log level: #{level}" unless LOG_LEVELS.value?(level)
183
170
 
184
171
  level
185
172
  else
186
- LOG_LEVELS.fetch(level.to_s.downcase.to_sym) { raise ArgumentError, "invalid log level: #{level}" }
173
+ LOG_LEVELS.fetch(level.to_s.downcase.to_sym) { raise InvalidOption, "invalid log level: #{level}" }
187
174
  end
188
175
  end
189
176
  end
@@ -32,15 +32,13 @@ module Html2rss
32
32
  end
33
33
  end
34
34
 
35
- module_function
36
-
37
35
  ENV_VAR = 'BOTASAURUS_SCRAPER_URL'
38
36
  private_constant :ENV_VAR
39
37
 
40
38
  ##
41
39
  # @param sample_url [String, nil] optional URL for a sample scrape smoke test
42
40
  # @return [Result]
43
- def call(sample_url: nil) # rubocop:disable Metrics/AbcSize
41
+ def self.call(sample_url: nil) # rubocop:disable Metrics/AbcSize
44
42
  env = env_check
45
43
  checks = [env_check_record(env)]
46
44
  return missing_env_failure(checks) unless checks.first.ok
@@ -57,7 +55,7 @@ module Html2rss
57
55
 
58
56
  ##
59
57
  # @return [Hash{Symbol => Object}]
60
- def env_check
58
+ def self.env_check
61
59
  base = ENV.fetch(ENV_VAR, '').strip
62
60
  {
63
61
  configured: MCP::Runtime.botasaurus_configured?,
@@ -69,71 +67,69 @@ module Html2rss
69
67
  ##
70
68
  # @param base_url [String]
71
69
  # @return [Hash{Symbol => Object}]
72
- def health_check(base_url)
73
- endpoint = "#{Url.for_channel(base_url).to_s.chomp('/')}/health"
74
- response = health_session.get(endpoint)
70
+ def self.health_check(base_url)
71
+ response = health_session.get("#{Url.for_channel(base_url).to_s.chomp('/')}/health")
75
72
  raise response.error if response.is_a?(HTTPX::ErrorResponse)
76
73
 
77
74
  body = JSON.parse(response.body.to_s)
78
- { ok: response.status == 200, status: response.status, version: body['version'] }
75
+ status = response.status
76
+ { ok: status == 200, status:, version: body['version'] }
77
+ rescue Html2rss::ContractError
78
+ raise
79
79
  rescue StandardError => error
80
80
  { ok: false, error: "#{error.class}: #{error.message}" }
81
81
  end
82
82
 
83
- def health_session
83
+ def self.health_session
84
84
  HTTPX.with(timeout: { operation_timeout: 5, request_timeout: 5, total_request_timeout: 5 },
85
85
  resolver_class: :system)
86
86
  end
87
- module_function :health_session
88
87
  private_class_method :health_session
89
88
 
90
89
  ##
91
90
  # @param url [String]
92
91
  # @return [Hash{Symbol => Object}]
93
- def sample_scrape(url)
92
+ def self.sample_scrape(url)
94
93
  wire = PageRecon::Diagnostics.call(url:, strategy: :botasaurus).to_wire_h
95
94
  {
96
95
  ok: wire[:status].to_i.between?(200, 399),
97
96
  status: wire[:status],
98
97
  articles_count: wire[:articles_count]
99
98
  }
100
- rescue StandardError => error
99
+ rescue Html2rss::ContractError
100
+ raise
101
+ rescue Html2rss::Error => error
101
102
  { ok: false, error: "#{error.class}: #{error.message}" }
102
103
  end
103
104
 
104
- def env_check_record(env = env_check)
105
+ def self.env_check_record(env = env_check)
105
106
  Check.new(name: :env, ok: env[:configured], detail: { var: env[:var], base_url_set: !env[:base_url].nil? })
106
107
  end
107
- module_function :env_check_record
108
108
  private_class_method :env_check_record
109
109
 
110
- def health_check_record(base_url)
110
+ def self.health_check_record(base_url)
111
111
  return Check.new(name: :health, ok: false, detail: { error: 'missing base URL' }) unless base_url
112
112
 
113
113
  health = health_check(base_url)
114
114
  Check.new(name: :health, ok: health[:ok], detail: health)
115
115
  end
116
- module_function :health_check_record
117
116
  private_class_method :health_check_record
118
117
 
119
- def sample_scrape_record(url)
118
+ def self.sample_scrape_record(url)
120
119
  sample = sample_scrape(url)
121
120
  Check.new(name: :sample_scrape, ok: sample[:ok], detail: sample)
122
121
  end
123
- module_function :sample_scrape_record
124
122
  private_class_method :sample_scrape_record
125
123
 
126
- def missing_env_failure(checks)
124
+ def self.missing_env_failure(checks)
127
125
  Log.info("doctor botasaurus: #{ENV_VAR} unset")
128
126
  failure(checks, "Set #{ENV_VAR} to the scrape-api base URL.")
129
127
  end
130
- module_function :missing_env_failure
131
128
  private_class_method :missing_env_failure
132
129
 
133
- def failure(checks, message)
130
+ def self.failure(checks, message)
134
131
  Result.new(ok: false, checks:, message:)
135
132
  end
136
- module_function :failure
137
133
  private_class_method :failure
138
134
  end
139
135
  end
@@ -39,11 +39,21 @@ module Html2rss
39
39
  # @return [Hash{String => Object}, nil] allowlisted upstream transport telemetry when attached
40
40
  attr_accessor :transport_meta
41
41
 
42
+ ##
43
+ # Stable failure classification, overridden per error class.
44
+ #
45
+ # @return [Symbol] one of +:timeout+, +:blocked+, +:connection+, +:invalid_url+, +:budget+, +:other+
46
+ def kind = :other
47
+
48
+ ##
49
+ # @return [Boolean] whether repeating the same request may succeed (derived from {#kind})
50
+ def retryable? = ErrorKind[kind].retryable?
51
+
42
52
  # @return [String, nil] scrape-api timeout stage when known (overridden by RequestTimedOut)
43
- def timeout_phase = nil
53
+ def timeout_phase; end
44
54
 
45
55
  # @return [Symbol, nil] surface classification when known (overridden by NoFeedItemsExtracted)
46
- def surface_category = nil
56
+ def surface_category; end
47
57
 
48
58
  ##
49
59
  # Sparse telemetry hash for observability consumers.
@@ -59,6 +69,9 @@ module Html2rss
59
69
  end
60
70
  end
61
71
 
72
+ # Raised when a caller-supplied option value is malformed or out of range.
73
+ class InvalidOption < Error; end
74
+
62
75
  # Raised when auto fallback exhausts all concrete tiers and extractors find no feed items.
63
76
  class NoFeedItemsExtracted < Error
64
77
  # Categories that append shared surface guidance to the empty-feed message.
@@ -87,7 +100,8 @@ module Html2rss
87
100
 
88
101
  def base_message
89
102
  summaries = attempts.map do |attempt|
90
- details = attempt[:items_count].nil? ? "#{attempt[:error_class]} error" : "#{attempt[:items_count]} items"
103
+ count = attempt[:items_count]
104
+ details = count.nil? ? "#{attempt[:error_class]} error" : "#{count} items"
91
105
  "#{attempt[:strategy]} (#{details})"
92
106
  end.join(', ')
93
107
 
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Html2rss
4
+ ErrorKind = Data.define(:name, :retryable)
5
+
6
+ ##
7
+ # Closed classification of {Html2rss::Error} failures, so consumers branch on a stable
8
+ # symbol instead of exception class names. Read it through {Html2rss::Error#kind}.
9
+ class ErrorKind
10
+ # Frozen name → instance table. Only transient transport failures are retryable.
11
+ ALL = {
12
+ timeout: true,
13
+ blocked: false,
14
+ connection: true,
15
+ invalid_url: false,
16
+ budget: false,
17
+ other: false
18
+ }.to_h { |name, retryable| [name, new(name:, retryable:)] }.freeze
19
+ private_class_method :new
20
+
21
+ ##
22
+ # @param name [Symbol, String] one of +ALL.keys+
23
+ # @return [ErrorKind]
24
+ # @raise [InvalidOption] when the name is not a known kind
25
+ def self.[](name) = ALL.fetch(name.to_sym) { raise InvalidOption, "unknown error kind: #{name.inspect}" }
26
+
27
+ ##
28
+ # @return [Boolean] whether repeating the same request may succeed
29
+ def retryable? = retryable
30
+
31
+ ##
32
+ # @return [Symbol]
33
+ def to_sym = name
34
+
35
+ ##
36
+ # @return [String]
37
+ def to_s = name.to_s
38
+ end
39
+ end
@@ -5,11 +5,15 @@ module Html2rss
5
5
  ##
6
6
  # Presentation rules for rendering an {Html2rss::Article} into feed wire formats.
7
7
  #
8
- # Owns description enrichment ({Html::Rendering::DescriptionBuilder}) and RSS
9
- # non-image enclosure selection. {Article} keeps raw extracted fields only.
8
+ # Owns description enrichment ({Html::Rendering::DescriptionBuilder}), RSS enclosure
9
+ # choice and the media-feed rule. {Article} keeps raw extracted fields only; media-file
10
+ # facts live on {Article::Enclosure}.
10
11
  module ItemPresentation
11
12
  class << self
12
13
  ##
14
+ # The item body: sanitized description text after at most one media element, the RSS enclosure
15
+ # ({.rss_enclosure_for}) when it is audio or video, else the item image ({Html::Rendering::DescriptionBuilder}).
16
+ #
13
17
  # @param article [Html2rss::Article]
14
18
  # @return [String, nil] sanitized description HTML/text for feed item bodies
15
19
  def description_for(article)
@@ -17,24 +21,33 @@ module Html2rss
17
21
  base: article.description,
18
22
  title: article.title,
19
23
  url: article.url,
20
- enclosures: article.enclosures,
24
+ enclosure: rss_enclosure_for(article),
21
25
  image: article.image
22
26
  ).call
23
27
  end
24
28
 
25
29
  ##
26
- # First non-image enclosure for RSS +enclosure+ (images stay on description / JSON Feed).
30
+ # The one RSS +enclosure+ of an item: its first audio or video file, else its first downloadable
31
+ # document or archive ({Article::Enclosure#document?}: PDF, ZIP). Images stay on the description /
32
+ # JSON Feed; stream playlists, Flash players and watch or player pages ({Article::Enclosure#player_page?})
33
+ # are never one.
27
34
  #
28
35
  # @param article [Html2rss::Article]
29
36
  # @return [Html2rss::Article::Enclosure, nil]
30
37
  def rss_enclosure_for(article)
31
- article.enclosures.find { |enc| !image_mime_type?(enc.type) }
38
+ enclosures = article.enclosures
39
+ enclosures.find(&:media?) || enclosures.find(&:document?)
32
40
  end
33
41
 
34
- private
35
-
36
- def image_mime_type?(type)
37
- type.to_s.downcase.start_with?('image/')
42
+ ##
43
+ # A media feed (podcast) is one where more than half of the items carry an audio or video
44
+ # RSS enclosure; only such a feed gets podcast (+itunes:+) tags.
45
+ #
46
+ # @param articles [Array<Html2rss::Article>]
47
+ # @return [Boolean]
48
+ def media_feed?(articles)
49
+ media_items = articles.count { |article| article.enclosures.any?(&:media?) }
50
+ media_items * 2 > articles.size
38
51
  end
39
52
  end
40
53
  end
@@ -103,7 +103,8 @@ module Html2rss
103
103
  {
104
104
  url: enclosure.url.to_s,
105
105
  mime_type: enclosure.type,
106
- size_in_bytes: size&.positive? ? size : nil
106
+ size_in_bytes: (size if size.positive?),
107
+ duration_in_seconds: enclosure.duration
107
108
  }.compact
108
109
  end
109
110
  end
@@ -16,7 +16,7 @@ module Html2rss
16
16
  # @param user_comment [String] required generator comment (from {Status})
17
17
  # @param feed_url [String, nil] optional absolute self URL for the feed (JSON Feed feed_url)
18
18
  def initialize(channel:, articles:, user_comment:, feed_url: nil)
19
- raise ArgumentError, 'user_comment must be a non-blank String' if blank_string?(user_comment)
19
+ raise InvalidOption, 'user_comment must be a non-blank String' if blank_string?(user_comment)
20
20
 
21
21
  @channel = channel
22
22
  @articles = articles
@@ -42,7 +42,7 @@ module Html2rss
42
42
 
43
43
  def normalize_feed_url(value)
44
44
  return if value.nil?
45
- raise ArgumentError, 'feed_url must be a String or nil' unless value.is_a?(String)
45
+ raise InvalidOption, 'feed_url must be a String or nil' unless value.is_a?(String)
46
46
  return if value.strip.empty?
47
47
 
48
48
  Url.from_absolute(value).to_s
@@ -43,9 +43,9 @@ module Html2rss
43
43
  # @param type [String] MIME type (`text/css` or `text/xsl`)
44
44
  # @param media [String] media query hint for the stylesheet
45
45
  def initialize(href:, type:, media: 'all')
46
- raise ArgumentError, 'stylesheet.href must be a String' unless href.is_a?(String)
47
- raise ArgumentError, 'stylesheet.type invalid' unless TYPES.include?(type)
48
- raise ArgumentError, 'stylesheet.media must be a String' unless media.is_a?(String)
46
+ raise InvalidOption, 'stylesheet.href must be a String' unless href.is_a?(String)
47
+ raise InvalidOption, 'stylesheet.type invalid' unless TYPES.include?(type)
48
+ raise InvalidOption, 'stylesheet.media must be a String' unless media.is_a?(String)
49
49
 
50
50
  @href = href
51
51
  @type = type