html2rss 0.32.3 → 0.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. checksums.yaml +4 -4
  2. data/lib/html2rss/article/enclosure.rb +2 -6
  3. data/lib/html2rss/auto_source/cleanup.rb +1 -3
  4. data/lib/html2rss/auto_source/scraper/html.rb +2 -8
  5. data/lib/html2rss/auto_source/scraper/json_document.rb +2 -3
  6. data/lib/html2rss/auto_source/scraper/json_state/article_normalizer.rb +128 -128
  7. data/lib/html2rss/auto_source/scraper/json_state/candidate_detector.rb +37 -37
  8. data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +123 -123
  9. data/lib/html2rss/auto_source/scraper/json_state/value_finder.rb +34 -34
  10. data/lib/html2rss/auto_source/scraper/json_state.rb +2 -6
  11. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +32 -2
  12. data/lib/html2rss/auto_source/scraper/microdata.rb +228 -245
  13. data/lib/html2rss/auto_source/scraper/microformats2.rb +9 -1
  14. data/lib/html2rss/auto_source/scraper/native_feed.rb +1 -1
  15. data/lib/html2rss/auto_source/scraper/schema/category_extractor.rb +7 -2
  16. data/lib/html2rss/auto_source/scraper/schema/item_list.rb +2 -2
  17. data/lib/html2rss/auto_source/scraper/schema/thing.rb +98 -21
  18. data/lib/html2rss/auto_source/scraper/schema.rb +49 -2
  19. data/lib/html2rss/auto_source/scraper/semantic_html.rb +12 -16
  20. data/lib/html2rss/auto_source/scraper/sitemap/parser.rb +1 -4
  21. data/lib/html2rss/auto_source/scraper/sitemap.rb +3 -11
  22. data/lib/html2rss/auto_source/scraper/wordpress_api/page_scope.rb +2 -2
  23. data/lib/html2rss/auto_source/scraper/wordpress_api/posts_endpoint.rb +1 -3
  24. data/lib/html2rss/auto_source/scraper/wordpress_api.rb +4 -4
  25. data/lib/html2rss/auto_source/scraper/xhr_articles.rb +1 -1
  26. data/lib/html2rss/auto_source/scraper.rb +0 -11
  27. data/lib/html2rss/auto_source/segmenter/cluster.rb +5 -11
  28. data/lib/html2rss/auto_source/segmenter/list.rb +25 -57
  29. data/lib/html2rss/auto_source/segmenter/semantic.rb +6 -13
  30. data/lib/html2rss/auto_source.rb +1 -1
  31. data/lib/html2rss/batch.rb +22 -32
  32. data/lib/html2rss/capture/README.md +1 -1
  33. data/lib/html2rss/capture.rb +7 -27
  34. data/lib/html2rss/cli/render.rb +97 -96
  35. data/lib/html2rss/cli/validate.rb +2 -4
  36. data/lib/html2rss/cli.rb +33 -27
  37. data/lib/html2rss/config/request_controls.rb +10 -2
  38. data/lib/html2rss/config/schema.rb +8 -10
  39. data/lib/html2rss/config/selectors_validator.rb +1 -0
  40. data/lib/html2rss/config/validator.rb +3 -3
  41. data/lib/html2rss/config.rb +1 -1
  42. data/lib/html2rss/defaults.rb +5 -16
  43. data/lib/html2rss/doctor/botasaurus.rb +11 -19
  44. data/lib/html2rss/error.rb +54 -1
  45. data/lib/html2rss/feed_builder/rss.rb +1 -1
  46. data/lib/html2rss/feed_pipeline/README.md +1 -1
  47. data/lib/html2rss/feed_pipeline/auto_fallback.rb +58 -55
  48. data/lib/html2rss/feed_resolution/candidate_generator.rb +3 -4
  49. data/lib/html2rss/feed_resolution/options.rb +3 -7
  50. data/lib/html2rss/feed_resolution/policy.rb +29 -33
  51. data/lib/html2rss/feed_resolution/probe.rb +3 -3
  52. data/lib/html2rss/feed_resolution/scorer.rb +29 -29
  53. data/lib/html2rss/feed_resolution.rb +5 -9
  54. data/lib/html2rss/feed_result.rb +4 -3
  55. data/lib/html2rss/hash_util.rb +91 -90
  56. data/lib/html2rss/html/article_extractor/category_extractor.rb +6 -9
  57. data/lib/html2rss/html/article_extractor/date_extractor.rb +1 -1
  58. data/lib/html2rss/html/article_extractor/enclosure_extractor.rb +1 -3
  59. data/lib/html2rss/html/article_extractor/id_generator.rb +1 -1
  60. data/lib/html2rss/html/probe.rb +51 -51
  61. data/lib/html2rss/link_destination/path_classifier.rb +11 -33
  62. data/lib/html2rss/mcp/config_argument.rb +9 -2
  63. data/lib/html2rss/mcp/contract.rb +7 -5
  64. data/lib/html2rss/mcp/outcome.rb +4 -5
  65. data/lib/html2rss/mcp/runtime.rb +17 -24
  66. data/lib/html2rss/mcp/server/tools.rb +9 -9
  67. data/lib/html2rss/option.rb +48 -0
  68. data/lib/html2rss/page_recon/diagnostics.rb +7 -9
  69. data/lib/html2rss/page_recon.rb +0 -4
  70. data/lib/html2rss/recon.rb +10 -12
  71. data/lib/html2rss/request_service/blocked_surface.rb +3 -7
  72. data/lib/html2rss/request_service/botasaurus_strategy.rb +2 -2
  73. data/lib/html2rss/request_service/budget.rb +3 -24
  74. data/lib/html2rss/request_service/compressed_body.rb +36 -48
  75. data/lib/html2rss/request_service/context.rb +1 -3
  76. data/lib/html2rss/request_service/local_file_strategy.rb +3 -1
  77. data/lib/html2rss/request_service/network_guard.rb +3 -2
  78. data/lib/html2rss/request_service/policy.rb +8 -26
  79. data/lib/html2rss/request_service/response.rb +3 -6
  80. data/lib/html2rss/request_service/strategy.rb +2 -2
  81. data/lib/html2rss/request_service.rb +26 -2
  82. data/lib/html2rss/request_session/pager/base.rb +3 -3
  83. data/lib/html2rss/request_session/pager/custom_selector.rb +4 -4
  84. data/lib/html2rss/request_session/pager/offset.rb +1 -7
  85. data/lib/html2rss/request_session/pager/rel_next.rb +1 -1
  86. data/lib/html2rss/request_session/pager/url_template.rb +1 -7
  87. data/lib/html2rss/scoring/engine.rb +4 -32
  88. data/lib/html2rss/scoring/link_resolver.rb +2 -7
  89. data/lib/html2rss/selectors/extractors/href.rb +2 -4
  90. data/lib/html2rss/selectors/json_xml.rb +33 -34
  91. data/lib/html2rss/selectors/post_processors/gsub.rb +2 -3
  92. data/lib/html2rss/selectors/post_processors/parse_time.rb +4 -5
  93. data/lib/html2rss/selectors/schema_export.rb +60 -60
  94. data/lib/html2rss/selectors.rb +14 -18
  95. data/lib/html2rss/sst/node.rb +9 -3
  96. data/lib/html2rss/sst/normalizer.rb +25 -24
  97. data/lib/html2rss/sst/text.rb +53 -57
  98. data/lib/html2rss/status.rb +23 -30
  99. data/lib/html2rss/syndication/alternate_links.rb +39 -0
  100. data/lib/html2rss/syndication/discovery.rb +117 -187
  101. data/lib/html2rss/syndication/parser.rb +92 -115
  102. data/lib/html2rss/test/enhance_audit.rb +13 -15
  103. data/lib/html2rss/test/policy.rb +9 -11
  104. data/lib/html2rss/test.rb +29 -48
  105. data/lib/html2rss/url.rb +53 -53
  106. data/lib/html2rss/version.rb +1 -1
  107. data/lib/html2rss.rb +2 -1
  108. metadata +4 -2
@@ -14,31 +14,24 @@ module Html2rss
14
14
  :botasaurus_configured
15
15
  )
16
16
 
17
- module_function
17
+ class << self
18
+ ##
19
+ # @return [Boolean] whether Botasaurus transport is configured in this process
20
+ def botasaurus_configured?
21
+ !ENV['BOTASAURUS_SCRAPER_URL'].to_s.strip.empty?
22
+ end
18
23
 
19
- ##
20
- # @return [Boolean] whether Botasaurus transport is configured in this process
21
- def botasaurus_configured?
22
- !ENV['BOTASAURUS_SCRAPER_URL'].to_s.strip.empty?
23
- end
24
-
25
- ##
26
- # @return [Snapshot] runtime capabilities and catalog identity for MCP clients
27
- def snapshot
28
- Snapshot.new(
29
- version: Html2rss::VERSION,
30
- mcp_contract_version: Contract::MCP_CONTRACT_VERSION,
31
- catalog_fingerprint: Contract.catalog_fingerprint,
32
- tools: Contract.catalog_tools,
33
- botasaurus_configured: botasaurus_configured?
34
- )
35
- end
36
-
37
- ##
38
- # @param strategy [String, Symbol, nil]
39
- # @return [Symbol]
40
- def coerce_strategy(strategy)
41
- (strategy || :auto).to_sym
24
+ ##
25
+ # @return [Snapshot] runtime capabilities and catalog identity for MCP clients
26
+ def snapshot
27
+ Snapshot.new(
28
+ version: Html2rss::VERSION,
29
+ mcp_contract_version: Contract::MCP_CONTRACT_VERSION,
30
+ catalog_fingerprint: Contract.catalog_fingerprint,
31
+ tools: Contract.catalog_tools,
32
+ botasaurus_configured: botasaurus_configured?
33
+ )
34
+ end
42
35
  end
43
36
  end
44
37
  end
@@ -36,9 +36,9 @@ module Html2rss
36
36
  input_schema: Contract::INSPECT_INPUT_SCHEMA,
37
37
  annotations: Contract::ANNOTATIONS_OPEN_WORLD,
38
38
  call: lambda { |url:, strategy: 'auto', **|
39
- Outcome.inspect(
39
+ Outcome.inspect_report(
40
40
  report: PageRecon::Diagnostics.call(
41
- url:, strategy: Runtime.coerce_strategy(strategy), deep: false
41
+ url:, strategy: Option.strategy(strategy), deep: false
42
42
  )
43
43
  )
44
44
  }
@@ -51,7 +51,7 @@ module Html2rss
51
51
  input_schema: Contract::RECON_INPUT_SCHEMA,
52
52
  annotations: Contract::ANNOTATIONS_OPEN_WORLD,
53
53
  call: lambda { |url:, strategy: 'auto', **|
54
- Outcome.recon(result: Html2rss.recon(url, strategy: Runtime.coerce_strategy(strategy)))
54
+ Outcome.recon(result: Html2rss.recon(url, strategy: Option.strategy(strategy)))
55
55
  }
56
56
  ),
57
57
  Tool.new(
@@ -63,7 +63,7 @@ module Html2rss
63
63
  annotations: Contract::ANNOTATIONS_OPEN_WORLD,
64
64
  call: lambda { |urls:, strategy: 'auto', concurrency: Batch::DEFAULT_CONCURRENCY, limit: 10, **|
65
65
  Outcome.batch_scrape(
66
- Batch.batch_scrape(urls:, strategy: Runtime.coerce_strategy(strategy), concurrency:, limit:)
66
+ Batch.batch_scrape(urls:, strategy: Option.strategy(strategy), concurrency:, limit:)
67
67
  )
68
68
  }
69
69
  ),
@@ -76,7 +76,7 @@ module Html2rss
76
76
  annotations: Contract::ANNOTATIONS_OPEN_WORLD,
77
77
  call: lambda { |urls:, strategy: 'auto', concurrency: Batch::DEFAULT_CONCURRENCY, **|
78
78
  Outcome.batch_inspect(
79
- Batch.batch_inspect(urls:, strategy: Runtime.coerce_strategy(strategy), concurrency:)
79
+ Batch.batch_inspect(urls:, strategy: Option.strategy(strategy), concurrency:)
80
80
  )
81
81
  }
82
82
  ),
@@ -89,7 +89,7 @@ module Html2rss
89
89
  annotations: Contract::ANNOTATIONS_OPEN_WORLD,
90
90
  call: lambda { |urls:, strategy: 'auto', concurrency: Batch::DEFAULT_CONCURRENCY, **|
91
91
  Outcome.batch_recon(
92
- Batch.batch_recon(urls:, strategy: Runtime.coerce_strategy(strategy), concurrency:)
92
+ Batch.batch_recon(urls:, strategy: Option.strategy(strategy), concurrency:)
93
93
  )
94
94
  }
95
95
  ),
@@ -136,7 +136,7 @@ module Html2rss
136
136
  compare_enhance: false, **kwargs|
137
137
  feed_config = ConfigArgument.parse(config:, yaml:).config
138
138
  test_args = { min_items:, strict_quality:, compare_enhance: }
139
- test_args[:strategy] = Runtime.coerce_strategy(kwargs[:strategy]) if kwargs.key?(:strategy)
139
+ test_args[:strategy] = Option.strategy(kwargs[:strategy]) if kwargs.key?(:strategy)
140
140
  Outcome.test(Html2rss.test(feed_config, **test_args))
141
141
  }
142
142
  ),
@@ -163,7 +163,7 @@ module Html2rss
163
163
  # @param items_selector [String, nil]
164
164
  # @return [Outcome]
165
165
  def scrape_outcome(url:, strategy: 'auto', limit: 25, items_selector: nil)
166
- wire = Batch.scrape_wire(url:, strategy: Runtime.coerce_strategy(strategy), limit:, items_selector:)
166
+ wire = Batch.scrape_wire(url:, strategy: Option.strategy(strategy), limit:, items_selector:)
167
167
  Outcome.scrape(
168
168
  items: wire[:items],
169
169
  requested_strategy: wire[:strategy],
@@ -189,7 +189,7 @@ module Html2rss
189
189
  # @return [Outcome]
190
190
  def capture_outcome(url:, strategy: 'auto', items_selector: nil, force: false, topics: nil, title: nil, # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists
191
191
  summary: nil, enhance: nil, limit: nil, max_redirects: nil, max_requests: nil)
192
- plan = Runtime.coerce_strategy(strategy)
192
+ plan = Option.strategy(strategy)
193
193
  result = Html2rss::Capture.build(
194
194
  url,
195
195
  strategy: plan,
@@ -0,0 +1,48 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Html2rss
4
+ ##
5
+ # Edge validation for caller-supplied option values (public API, CLI, MCP).
6
+ # Everything past these checks may trust its types.
7
+ module Option
8
+ class << self
9
+ ##
10
+ # @param name [Symbol] option name used in the error message
11
+ # @param value [Object] candidate value
12
+ # @param min [Integer] inclusive lower bound
13
+ # @param max [Integer, nil] inclusive upper bound
14
+ # @param optional [Boolean] whether +nil+ passes through
15
+ # @return [Integer, nil] +value+ when valid
16
+ # @raise [InvalidOption] when +value+ is not an Integer within bounds
17
+ def integer(name, value, min:, max: nil, optional: false)
18
+ return value if value.nil? && optional
19
+ return value if value.is_a?(Integer) && (min..max).cover?(value)
20
+
21
+ raise InvalidOption, "#{name} must be an Integer in #{min}..#{max}, got #{value.inspect}"
22
+ end
23
+
24
+ ##
25
+ # @param value [Symbol, String, nil] strategy plan name (+nil+ means +:auto+)
26
+ # @return [Symbol] the plan name
27
+ # @raise [InvalidOption] when +value+ is not a registered plan
28
+ def strategy(value)
29
+ name = value.nil? ? FeedPipeline::StrategyPlan::AUTO_NAME : value
30
+ return name.to_sym if FeedPipeline::StrategyPlan.valid?(name)
31
+
32
+ accepted = FeedPipeline::StrategyPlan.accepted_names.join(', ')
33
+ raise InvalidOption, "strategy must be one of #{accepted}, got #{value.inspect}"
34
+ end
35
+
36
+ ##
37
+ # @param name [Symbol] option name used in the error message
38
+ # @param value [Object] candidate list
39
+ # @return [Array] +value+ when it is a non-empty Array
40
+ # @raise [InvalidOption] otherwise
41
+ def list(name, value)
42
+ return value if value.is_a?(Array) && value.any?
43
+
44
+ raise InvalidOption, "#{name} must be a non-empty Array, got #{value.inspect}"
45
+ end
46
+ end
47
+ end
48
+ end
@@ -34,14 +34,12 @@ module Html2rss
34
34
  JS_SHELL_MIN_BODY_BYTES = 8_192
35
35
  private_constant :JS_SHELL_MIN_BODY_BYTES
36
36
 
37
- module_function
38
-
39
37
  ##
40
38
  # @param url [String]
41
39
  # @param strategy [String, Symbol]
42
40
  # @param deep [Boolean] when true and strategy is auto, one Botasaurus hop if configured
43
41
  # @return [Report]
44
- def call(url:, strategy: :auto, deep: false)
42
+ def self.call(url:, strategy: :auto, deep: false)
45
43
  probe = PageRecon.probe(url, strategy: resolve_inspect_strategy(strategy, deep:))
46
44
  recon = probe.result
47
45
  response = probe.response
@@ -56,10 +54,10 @@ module Html2rss
56
54
  # @param strategy [Symbol, String]
57
55
  # @param concurrency [Integer]
58
56
  # @return [Array<Report>]
59
- def batch(urls:, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY)
57
+ def self.batch(urls:, strategy: :auto, concurrency: Batch::DEFAULT_CONCURRENCY)
60
58
  Batch.map(Array(urls), concurrency:) do |url|
61
59
  call(url:, strategy:)
62
- rescue StandardError => error
60
+ rescue Html2rss::Error => error
63
61
  error_report(url, error)
64
62
  end
65
63
  end
@@ -67,7 +65,7 @@ module Html2rss
67
65
  ##
68
66
  # @param response [Html2rss::RequestService::Response]
69
67
  # @return [Hash] redacted XHR capture diagnostics (no query strings)
70
- def xhr_capture_info(response)
68
+ def self.xhr_capture_info(response)
71
69
  captured = response.captured_responses
72
70
  {
73
71
  count: captured.size,
@@ -79,7 +77,7 @@ module Html2rss
79
77
  ##
80
78
  # @param entry [Hash] captured response hash
81
79
  # @return [String, nil] scheme+host+path only
82
- def redacted_endpoint(entry)
80
+ def self.redacted_endpoint(entry)
83
81
  raw = entry['url'] || entry[:url]
84
82
  return unless raw
85
83
 
@@ -94,7 +92,7 @@ module Html2rss
94
92
  ##
95
93
  # @param entry [Hash] captured response hash
96
94
  # @return [Boolean]
97
- def xhr_candidate_articles?(entry)
95
+ def self.xhr_candidate_articles?(entry)
98
96
  body = entry['body'] || entry[:body]
99
97
  return false unless body.is_a?(String)
100
98
 
@@ -107,7 +105,7 @@ module Html2rss
107
105
  ##
108
106
  # @param parsed [Object] parsed response body
109
107
  # @return [Array<String>, Hash]
110
- def scraper_info(parsed)
108
+ def self.scraper_info(parsed)
111
109
  return { error: 'Response is not HTML' } unless parsed.is_a?(Nokogiri::HTML::Document)
112
110
 
113
111
  begin
@@ -158,8 +158,6 @@ module Html2rss
158
158
  def self.discover_segments(sst, url)
159
159
  link_resolver = Scoring::LinkResolver.new(url)
160
160
  AutoSource::Segmenter.call(sst, base_url: url, strategy: :list, link_resolver:)
161
- rescue StandardError
162
- []
163
161
  end
164
162
 
165
163
  ##
@@ -268,8 +266,6 @@ module Html2rss
268
266
 
269
267
  def sst_document
270
268
  Html2rss::SST::Normalizer.call(response.body)
271
- rescue ArgumentError
272
- nil
273
269
  end
274
270
 
275
271
  def segment_stats(sst, page_url)
@@ -9,8 +9,6 @@ module Html2rss
9
9
  # Service that runs reconnaissance on a URL to discover redirect chains,
10
10
  # native RSS/Atom feeds, surface categorization, and emit an actionable verdict.
11
11
  module Recon # rubocop:disable Metrics/ModuleLength
12
- module_function
13
-
14
12
  Verdict = Data.define(:name)
15
13
 
16
14
  ##
@@ -106,12 +104,12 @@ module Html2rss
106
104
  # @option options [Integer, nil] :max_redirects optional maximum redirects
107
105
  # @option options [Integer, nil] :max_requests optional request budget
108
106
  # @return [Html2rss::Recon::Result]
109
- def call(url, strategy: :auto, cache_dir: nil, cache_mutex: nil, **) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
107
+ def self.call(url, strategy: :auto, cache_dir: nil, cache_mutex: nil, **) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
110
108
  url_obj = Url.from_absolute(url)
111
109
 
112
110
  begin
113
111
  probe = PageRecon.probe(url_obj, strategy:, **)
114
- rescue StandardError => error
112
+ rescue Html2rss::Error => error
115
113
  return error_result(url_obj, error)
116
114
  end
117
115
 
@@ -149,7 +147,7 @@ module Html2rss
149
147
  # @option options [Integer, nil] :max_redirects optional maximum redirects
150
148
  # @option options [Integer, nil] :max_requests optional request budget
151
149
  # @return [Array<Html2rss::Recon::Result>]
152
- def batch(urls, strategy: :auto, cache_dir: nil, max_threads: 5, **options)
150
+ def self.batch(urls, strategy: :auto, cache_dir: nil, max_threads: 5, **options)
153
151
  url_list = urls.map { |u| u.to_s.strip }.reject(&:empty?)
154
152
  return [] if url_list.empty?
155
153
 
@@ -166,7 +164,7 @@ module Html2rss
166
164
  # @param session [Html2rss::RequestSession]
167
165
  # @param response [Html2rss::RequestService::Response]
168
166
  # @return [Array(Html2rss::Url, nil), String, nil)] feed URL and optional discovery error note
169
- def find_native_feed(url_obj, session, response)
167
+ def self.find_native_feed(url_obj, session, response)
170
168
  feed = Syndication::Discovery.best_feed_url(
171
169
  page_url: url_obj,
172
170
  request_session: session,
@@ -174,12 +172,12 @@ module Html2rss
174
172
  html: response.body
175
173
  )
176
174
  [feed, nil]
177
- rescue StandardError => error
175
+ rescue Html2rss::Error => error
178
176
  [nil, "discovery_error=#{error.class}: #{error.message}"]
179
177
  end
180
178
  private_class_method :find_native_feed
181
179
 
182
- def build_notes(recon, native_feed, response)
180
+ def self.build_notes(recon, native_feed, response)
183
181
  notes = []
184
182
  notes << "native_rss=#{native_feed}" if native_feed
185
183
  if recon.scheme_downgrade
@@ -192,7 +190,7 @@ module Html2rss
192
190
  end
193
191
  private_class_method :build_notes
194
192
 
195
- def determine_verdict(recon, native_feed)
193
+ def self.determine_verdict(recon, native_feed)
196
194
  return Verdict[:drop] if recon.status.nil? || recon.status >= 400 || recon.scheme_downgrade
197
195
  return Verdict[:defer] if native_feed
198
196
 
@@ -203,7 +201,7 @@ module Html2rss
203
201
  end
204
202
  private_class_method :determine_verdict
205
203
 
206
- def error_result(url_obj, error) # rubocop:disable Metrics/MethodLength
204
+ def self.error_result(url_obj, error) # rubocop:disable Metrics/MethodLength
207
205
  Result.new(
208
206
  requested_url: url_obj,
209
207
  final_url: url_obj,
@@ -226,14 +224,14 @@ module Html2rss
226
224
  # @param cache_dir [String]
227
225
  # @param cache_mutex [Mutex, nil]
228
226
  # @return [void]
229
- def cache_html_body(body, url_obj, cache_dir, cache_mutex)
227
+ def self.cache_html_body(body, url_obj, cache_dir, cache_mutex)
230
228
  host = url_obj.host.to_s.delete_prefix('www.')
231
229
  host = 'snapshot' if host.empty?
232
230
  digest = Digest::SHA256.hexdigest(url_obj.to_s)[0, 12]
233
231
  file_path = File.join(cache_dir, "#{host}-#{digest}.html")
234
232
  writer = -> { File.write(file_path, body) }
235
233
  cache_mutex ? cache_mutex.synchronize(&writer) : writer.call
236
- rescue StandardError => error
234
+ rescue SystemCallError, IOError => error
237
235
  Log.debug("Recon cache failed: #{error.message}")
238
236
  end
239
237
  private_class_method :cache_html_body
@@ -79,10 +79,10 @@ module Html2rss
79
79
  end
80
80
 
81
81
  def self.interstitial_signature_match?(body, signature)
82
- min_matches = signature.fetch(:min_matches, 1)
82
+ min_matches = signature[:min_matches]
83
83
  matches = 0
84
84
 
85
- signature.fetch(:patterns).each do |pattern|
85
+ signature[:patterns].each do |pattern|
86
86
  matches += 1 if pattern.match?(body)
87
87
  return true if matches >= min_matches
88
88
  end
@@ -91,11 +91,7 @@ module Html2rss
91
91
  end
92
92
  private_class_method :interstitial_signature_match?
93
93
 
94
- def self.normalize_body(body)
95
- body.to_s.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '')
96
- rescue Encoding::CompatibilityError, Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
97
- body.to_s.force_encoding(Encoding::UTF_8).scrub
98
- end
94
+ def self.normalize_body(body) = body.to_s.scrub
99
95
  private_class_method :normalize_body
100
96
  end
101
97
  end
@@ -65,7 +65,7 @@ module Html2rss
65
65
  return ctx.url if final_url.nil?
66
66
 
67
67
  Html2rss::Url.from_absolute(final_url)
68
- rescue ArgumentError
68
+ rescue Html2rss::Url::InvalidUrlError
69
69
  ctx.url
70
70
  end
71
71
 
@@ -117,7 +117,7 @@ module Html2rss
117
117
  raise BotasaurusConfigurationError, 'BOTASAURUS_SCRAPER_URL is required for strategy=botasaurus.'
118
118
  end
119
119
  Html2rss::Url.for_channel(configured)
120
- rescue ArgumentError => error
120
+ rescue Html2rss::Url::InvalidUrlError => error
121
121
  raise BotasaurusConfigurationError, "BOTASAURUS_SCRAPER_URL is invalid: #{error.message}"
122
122
  end
123
123
  end
@@ -11,7 +11,7 @@ module Html2rss
11
11
  # @param max_requests [Integer] maximum HTTP request slots
12
12
  # @param total_timeout_seconds [Integer, nil] wall-clock timeout for the feed build
13
13
  def initialize(max_requests:, total_timeout_seconds: nil)
14
- validate_slot_limits!(max_requests:)
14
+ Option.integer(:max_requests, max_requests, min: 1)
15
15
 
16
16
  @remaining_requests = max_requests
17
17
  @start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
@@ -25,33 +25,20 @@ module Html2rss
25
25
  # @return [Integer] remaining request slots after consumption
26
26
  # @raise [RequestBudgetExceeded] if no request slots remain
27
27
  def consume!
28
- consume_request!
29
- end
30
-
31
- ##
32
- # Consumes one HTTP request slot.
33
- #
34
- # @return [Integer] remaining request slots after consumption
35
- # @raise [RequestBudgetExceeded] if no request slots remain
36
- def consume_request!
37
28
  @mutex.synchronize do
38
29
  raise RequestBudgetExceeded, 'Request budget exhausted' if @remaining_requests.zero?
39
30
 
40
31
  @remaining_requests -= 1
41
32
  end
42
33
  end
43
-
44
- ##
45
- # @return [Integer] HTTP request slots still available
46
- def remaining
47
- remaining_requests
48
- end
34
+ alias consume_request! consume!
49
35
 
50
36
  ##
51
37
  # @return [Integer] HTTP request slots still available
52
38
  def remaining_requests
53
39
  @mutex.synchronize { @remaining_requests }
54
40
  end
41
+ alias remaining remaining_requests
55
42
 
56
43
  ##
57
44
  # @return [Float, nil] the remaining timeout in seconds, or nil if not tracked
@@ -91,14 +78,6 @@ module Html2rss
91
78
  def effective_timeout_ms(fallback:)
92
79
  (effective_timeout_seconds(fallback:) * 1000).to_i
93
80
  end
94
-
95
- private
96
-
97
- def validate_slot_limits!(max_requests:)
98
- return if max_requests.is_a?(Integer) && max_requests.positive?
99
-
100
- raise ArgumentError, 'max_requests must be positive'
101
- end
102
81
  end
103
82
  end
104
83
  end
@@ -7,56 +7,58 @@ require 'zlib'
7
7
  module Html2rss
8
8
  class RequestService
9
9
  # Decodes gzip/brotli bodies when Content-Encoding is missing or the type is octet-stream.
10
- module CompressedBody
10
+ class CompressedBody
11
11
  # Gzip stream magic (RFC 1952).
12
12
  GZIP_MAGIC = "\x1F\x8B".b.freeze
13
13
  # HTML prefix for unlabeled brotli trial decode. Same object as {Response::HTML_BODY_SNIFF}.
14
14
  HTML_BODY_SNIFF = Response::HTML_BODY_SNIFF
15
15
 
16
- module_function
17
-
18
16
  # @param body [String, nil] raw response body
19
17
  # @param headers [Hash] response headers (Content-Encoding / Content-Type)
20
18
  # @return [String, nil] decoded body when compressed; otherwise the original body
21
- def decode(body, headers: {})
22
- return body if body.nil? || body.empty?
23
-
19
+ def self.decode(body, headers: {})
24
20
  raw = body.to_s.b
25
- decode_labeled(raw, headers) || decode_unlabeled(raw, headers) || body
21
+ new(raw, headers).decoded_body || body
22
+ end
23
+
24
+ # @param raw [String] binary response body
25
+ # @param headers [Hash] response headers (Content-Encoding / Content-Type)
26
+ def initialize(raw, headers)
27
+ @raw = raw
28
+ @headers = headers
26
29
  end
30
+ private_class_method :new
27
31
 
28
- def decode_labeled(raw, headers)
29
- encodings = content_encodings(headers)
32
+ # @return [String, nil] inflated body, or nil when the bytes are not a recognized stream
33
+ def decoded_body
34
+ decode_labeled || decode_unlabeled
35
+ end
36
+
37
+ private
38
+
39
+ def decode_labeled
40
+ encodings = content_encodings
30
41
  return if encodings.empty?
31
42
 
32
- encodings.reverse.reduce(raw) { |buffer, encoding| inflate(buffer, encoding) || buffer }
43
+ encodings.reverse.reduce(@raw) { |buffer, encoding| inflate(buffer, encoding) || buffer }
33
44
  end
34
- module_function :decode_labeled
35
- private_class_method :decode_labeled
36
45
 
37
- def decode_unlabeled(raw, headers)
38
- return uncompress_gzip(raw) if raw.start_with?(GZIP_MAGIC)
39
- return unless octet_stream?(headers)
46
+ def decode_unlabeled
47
+ return uncompress_gzip(@raw) if @raw.start_with?(GZIP_MAGIC)
48
+ return unless octet_stream?
40
49
 
41
- try_brotli(raw)
50
+ try_brotli(@raw)
42
51
  end
43
- module_function :decode_unlabeled
44
- private_class_method :decode_unlabeled
45
52
 
46
53
  def inflate(raw, encoding)
47
54
  case encoding
48
- when 'gzip'
49
- return raw unless raw.start_with?(GZIP_MAGIC)
50
-
51
- uncompress_gzip(raw)
55
+ when 'gzip' then uncompress_gzip(raw)
52
56
  when 'br' then try_brotli(raw, require_html: false)
53
57
  when 'deflate' then inflate_deflate(raw)
54
58
  end
55
59
  rescue Zlib::Error, Brotli::Error, ArgumentError
56
60
  nil
57
61
  end
58
- module_function :inflate
59
- private_class_method :inflate
60
62
 
61
63
  def inflate_deflate(raw)
62
64
  inflater = nil
@@ -67,51 +69,37 @@ module Html2rss
67
69
  ensure
68
70
  inflater&.close
69
71
  end
70
- module_function :inflate_deflate
71
- private_class_method :inflate_deflate
72
72
 
73
73
  def uncompress_gzip(raw)
74
- Zlib::GzipReader.wrap(StringIO.new(raw), encoding: 'ASCII-8BIT', &:read)
74
+ Zlib::GzipReader.wrap(StringIO.new(raw), &:read).b
75
75
  rescue Zlib::Error
76
76
  nil
77
77
  end
78
- module_function :uncompress_gzip
79
- private_class_method :uncompress_gzip
80
78
 
81
79
  def try_brotli(raw, require_html: true)
82
80
  inflated = Brotli.inflate(raw)
83
81
  return inflated unless require_html
84
- return inflated if inflated.b.match?(HTML_BODY_SNIFF)
85
82
 
86
- nil
83
+ inflated.b.match?(HTML_BODY_SNIFF) ? inflated : nil
87
84
  rescue Brotli::Error, ArgumentError
88
85
  nil
89
86
  end
90
- module_function :try_brotli
91
- private_class_method :try_brotli
92
87
 
93
- def content_encodings(headers)
94
- header(headers, 'content-encoding')
88
+ def content_encodings
89
+ header('content-encoding')
95
90
  .to_s
96
91
  .split(',')
97
- .filter_map { |val| (s = val.strip.downcase).empty? ? nil : s }
92
+ .map { |val| val.strip.downcase }
93
+ .reject(&:empty?)
98
94
  end
99
- module_function :content_encodings
100
- private_class_method :content_encodings
101
95
 
102
- def octet_stream?(headers)
103
- header(headers, 'content-type').to_s.downcase.include?('application/octet-stream')
96
+ def octet_stream?
97
+ header('content-type').to_s.downcase.include?('application/octet-stream')
104
98
  end
105
- module_function :octet_stream?
106
- private_class_method :octet_stream?
107
99
 
108
- def header(headers, name)
109
- headers.fetch(name) do
110
- headers.find { |key, _value| key.to_s.casecmp?(name) }&.last
111
- end
100
+ def header(name)
101
+ @headers.find { |key, _value| key.to_s.casecmp?(name) }&.last
112
102
  end
113
- module_function :header
114
- private_class_method :header
115
103
  end
116
104
  end
117
105
  end
@@ -87,9 +87,7 @@ module Html2rss
87
87
  end
88
88
 
89
89
  def normalize_request(request)
90
- normalized = HashUtil.deep_symbolize_keys(request, context: 'request')
91
- HashUtil.assert_symbol_keys!(normalized, context: 'request')
92
- normalized
90
+ HashUtil.deep_symbolize_keys(request, context: 'request')
93
91
  end
94
92
  end
95
93
  end
@@ -15,6 +15,9 @@ module Html2rss
15
15
 
16
16
  def skip_postflight? = true
17
17
 
18
+ # Local files never open a socket; keep filesystem errors as wire classes.
19
+ def connection_error?(_error) = false
20
+
18
21
  ##
19
22
  # @return [Response] the mock response wrapped around the file contents
20
23
  # @raise [ArgumentError] if the local file path is missing
@@ -22,7 +25,6 @@ module Html2rss
22
25
  def fetch
23
26
  file_path = ctx.request[:local_file_path]
24
27
  raise ArgumentError, 'Local file path is required for local_file strategy' unless file_path
25
- raise Errno::ENOENT, "File not found: #{file_path}" unless File.exist?(file_path)
26
28
 
27
29
  body = File.read(file_path)
28
30
  Response.new(
@@ -68,7 +68,7 @@ module Html2rss
68
68
  def validate_remote_ip!(ip:, url:)
69
69
  return if allow_private_networks?
70
70
 
71
- parsed_ip = ip && !ip.to_s.empty? ? parse_ip(ip) : nil
71
+ parsed_ip = parse_ip(ip.to_s)
72
72
  raise PrivateNetworkDenied, "Remote IP could not be validated for #{url}" unless parsed_ip
73
73
  return unless blocked_ip?(parsed_ip)
74
74
 
@@ -122,7 +122,8 @@ module Html2rss
122
122
  end
123
123
 
124
124
  def blocked_ip?(address)
125
- BLOCKED_IP_RANGES.any? { |range| range.include?(address) }
125
+ native = address.native
126
+ BLOCKED_IP_RANGES.any? { |range| range.include?(native) }
126
127
  end
127
128
  end
128
129
  end