html2rss 0.28.0 → 0.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +84 -27
  3. data/html2rss.gemspec +1 -4
  4. data/lib/html2rss/article.rb +1 -2
  5. data/lib/html2rss/auto_source/README.md +8 -8
  6. data/lib/html2rss/auto_source/cleanup.rb +92 -1
  7. data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +3 -5
  8. data/lib/html2rss/auto_source/scraper/meta_oembed.rb +15 -5
  9. data/lib/html2rss/auto_source/scraper/microdata.rb +8 -6
  10. data/lib/html2rss/auto_source/scraper/native_feed.rb +5 -2
  11. data/lib/html2rss/auto_source/scraper/schema.rb +39 -21
  12. data/lib/html2rss/auto_source/scraper.rb +2 -3
  13. data/lib/html2rss/auto_source/segmenter/cluster.rb +2 -3
  14. data/lib/html2rss/auto_source/segmenter/primary_link.rb +1 -2
  15. data/lib/html2rss/auto_source/segmenter.rb +1 -2
  16. data/lib/html2rss/auto_source.rb +25 -11
  17. data/lib/html2rss/batch.rb +169 -0
  18. data/lib/html2rss/capture/README.md +15 -6
  19. data/lib/html2rss/capture.rb +159 -16
  20. data/lib/html2rss/channel.rb +1 -2
  21. data/lib/html2rss/cli/probe_view.rb +66 -0
  22. data/lib/html2rss/cli/render.rb +124 -0
  23. data/lib/html2rss/cli/validate.rb +91 -0
  24. data/lib/html2rss/cli.rb +318 -193
  25. data/lib/html2rss/config/request_headers.rb +9 -16
  26. data/lib/html2rss/config/schema.rb +7 -5
  27. data/lib/html2rss/config/validator.rb +1 -0
  28. data/lib/html2rss/config.rb +68 -19
  29. data/lib/html2rss/doctor/botasaurus.rb +138 -0
  30. data/lib/html2rss/feed_builder/rss.rb +1 -2
  31. data/lib/html2rss/feed_pipeline/README.md +11 -11
  32. data/lib/html2rss/feed_pipeline/auto_fallback.rb +30 -11
  33. data/lib/html2rss/feed_pipeline/strategy_plan.rb +1 -1
  34. data/lib/html2rss/feed_pipeline.rb +27 -16
  35. data/lib/html2rss/feed_resolution/README.md +11 -11
  36. data/lib/html2rss/feed_resolution.rb +6 -12
  37. data/lib/html2rss/html/article_extractor/heading_extractor.rb +24 -14
  38. data/lib/html2rss/html/article_extractor.rb +4 -9
  39. data/lib/html2rss/html/feed_link.rb +4 -2
  40. data/lib/html2rss/html/navigator.rb +4 -39
  41. data/lib/html2rss/html/probe.rb +82 -0
  42. data/lib/html2rss/html/sst_article_extractor.rb +1 -2
  43. data/lib/html2rss/link_destination/noise_policy.rb +1 -2
  44. data/lib/html2rss/link_destination/path_classifier.rb +20 -19
  45. data/lib/html2rss/mcp/README.md +80 -0
  46. data/lib/html2rss/mcp/contract.rb +144 -18
  47. data/lib/html2rss/mcp/outcome/playbook.rb +122 -0
  48. data/lib/html2rss/mcp/outcome.rb +112 -42
  49. data/lib/html2rss/mcp/runtime.rb +45 -0
  50. data/lib/html2rss/mcp/server/tools.rb +277 -0
  51. data/lib/html2rss/mcp/server.rb +92 -233
  52. data/lib/html2rss/mcp.rb +3 -0
  53. data/lib/html2rss/page_recon/README.md +69 -0
  54. data/lib/html2rss/page_recon/diagnostics.rb +211 -0
  55. data/lib/html2rss/page_recon.rb +56 -2
  56. data/lib/html2rss/recon.rb +252 -0
  57. data/lib/html2rss/request_service/blocked_surface.rb +1 -0
  58. data/lib/html2rss/request_service/botasaurus_strategy.rb +32 -28
  59. data/lib/html2rss/request_service/compressed_body.rb +13 -8
  60. data/lib/html2rss/request_service/httpx_strategy.rb +228 -0
  61. data/lib/html2rss/request_service/policy.rb +1 -2
  62. data/lib/html2rss/request_service/response.rb +3 -4
  63. data/lib/html2rss/request_service/response_guard.rb +0 -16
  64. data/lib/html2rss/request_service/strategy.rb +20 -5
  65. data/lib/html2rss/request_service.rb +48 -85
  66. data/lib/html2rss/request_session.rb +1 -2
  67. data/lib/html2rss/scrape_target.rb +1 -1
  68. data/lib/html2rss/selectors/post_processors/sanitize_html.rb +2 -4
  69. data/lib/html2rss/selectors/post_processors.rb +0 -1
  70. data/lib/html2rss/selectors.rb +33 -10
  71. data/lib/html2rss/sst/attrs.rb +1 -2
  72. data/lib/html2rss/sst/normalizer.rb +49 -11
  73. data/lib/html2rss/status.rb +3 -6
  74. data/lib/html2rss/syndication/README.md +6 -6
  75. data/lib/html2rss/syndication/discovery.rb +2 -4
  76. data/lib/html2rss/test/enhance_audit.rb +192 -0
  77. data/lib/html2rss/test.rb +459 -0
  78. data/lib/html2rss/url.rb +1 -2
  79. data/lib/html2rss/version.rb +1 -1
  80. data/lib/html2rss.rb +157 -35
  81. data/schema/html2rss-config.schema.json +9 -29
  82. metadata +21 -56
  83. data/lib/html2rss/mcp/inspect.rb +0 -138
  84. data/lib/html2rss/request_service/faraday_strategy.rb +0 -229
  85. data/lib/html2rss/selectors/post_processors/html_to_markdown.rb +0 -59
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'faraday'
3
+ require 'httpx'
4
4
  require 'json'
5
5
  require 'securerandom'
6
6
 
@@ -11,7 +11,7 @@ module Html2rss
11
11
  class BotasaurusStrategy < Strategy
12
12
  # Content-Type negotiation for the scrape API transport hop.
13
13
  TRANSPORT_ACCEPT = 'application/json'
14
- # Disable compressed bodies so Faraday returns raw JSON without implicit decoding surprises.
14
+ # Disable compressed bodies so HTTPX returns raw JSON without implicit decoding surprises.
15
15
  TRANSPORT_ENCODING = 'identity'
16
16
  # Correlates each POST /scrape with botasaurus-scrape-api request logs.
17
17
  REQUEST_ID_HEADER = 'X-Request-Id'
@@ -28,10 +28,15 @@ module Html2rss
28
28
  def post_scrape_request
29
29
  request_id = SecureRandom.uuid
30
30
  Log.debug("#{self.class}: POST /scrape #{REQUEST_ID_HEADER}=#{request_id}")
31
- transport_response = client.post('/scrape', JSON.generate(contract.request_payload), post_headers(request_id))
32
- contract.parse_response(transport_response)
31
+ payload = JSON.generate(contract.request_payload)
32
+ response = client.post(scrape_endpoint, body: payload, headers: post_headers(request_id))
33
+ raise response.error if response.is_a?(HTTPX::ErrorResponse)
34
+
35
+ contract.parse_response(response)
33
36
  end
34
37
 
38
+ def scrape_endpoint = "#{scraper_base_url.to_s.chomp('/')}/scrape"
39
+
35
40
  def build_response(parsed_response)
36
41
  Response.new(
37
42
  body: parsed_response.html,
@@ -44,22 +49,14 @@ module Html2rss
44
49
  end
45
50
 
46
51
  def raise_from_error!(error)
47
- raise_if_challenge_blocked!(error)
48
- raise_if_timed_out!(error)
49
- raise BotasaurusServiceError, error.failure_message
50
- end
51
-
52
- def raise_if_challenge_blocked!(error)
53
- return unless error.challenge_block?
52
+ raise BlockedSurfaceDetected, "Blocked surface detected: #{error.challenge_message}" if error.challenge_block?
54
53
 
55
- raise BlockedSurfaceDetected, "Blocked surface detected: #{error.challenge_message}"
56
- end
57
-
58
- def raise_if_timed_out!(error)
59
- return unless error.timeout?
54
+ if error.timeout?
55
+ log_timeout!(reason: 'botasaurus_upstream')
56
+ raise RequestTimedOut.new(error.failure_message, timeout_phase: error.timeout_phase)
57
+ end
60
58
 
61
- log_timeout!(reason: 'botasaurus_upstream')
62
- raise RequestTimedOut.new(error.failure_message, timeout_phase: error.timeout_phase)
59
+ raise BotasaurusServiceError, error.failure_message
63
60
  end
64
61
 
65
62
  def response_url(final_url)
@@ -79,8 +76,18 @@ module Html2rss
79
76
  end
80
77
 
81
78
  def client
82
- # No :gzip middleware on this client — compression is for remote target fetches only.
83
- @client ||= Faraday.new(url: scraper_base_url.to_s, headers: client_headers, request: request_options)
79
+ timeout = attempt_timeout_seconds.to_f
80
+ @client ||= HTTPX.with(headers: client_headers, timeout: client_timeouts(timeout), resolver_class: :system)
81
+ end
82
+
83
+ def client_timeouts(timeout)
84
+ {
85
+ operation_timeout: timeout,
86
+ connect_timeout: [ctx.policy.connect_timeout_seconds, timeout].min,
87
+ read_timeout: [ctx.policy.read_timeout_seconds, timeout].min,
88
+ request_timeout: timeout,
89
+ total_request_timeout: timeout
90
+ }
84
91
  end
85
92
 
86
93
  def client_headers
@@ -92,14 +99,7 @@ module Html2rss
92
99
  end
93
100
 
94
101
  def post_headers(request_id)
95
- {
96
- 'Content-Type' => 'application/json',
97
- REQUEST_ID_HEADER => request_id
98
- }
99
- end
100
-
101
- def request_options
102
- { timeout: attempt_timeout_seconds.to_i }
102
+ { 'Content-Type' => 'application/json', REQUEST_ID_HEADER => request_id }
103
103
  end
104
104
 
105
105
  def attempt_timeout_seconds
@@ -123,6 +123,10 @@ module Html2rss
123
123
  def translate_connection_error(error)
124
124
  raise BotasaurusConnectionFailed, "Botasaurus connection failed: #{error.message}"
125
125
  end
126
+
127
+ def timeout_error?(error) = error.is_a?(HTTPX::TimeoutError) || super
128
+
129
+ def connection_error?(error) = error.is_a?(HTTPX::ConnectionError) || error.is_a?(HTTPX::TLSError) || super
126
130
  end
127
131
  end
128
132
  end
@@ -45,9 +45,12 @@ module Html2rss
45
45
 
46
46
  def inflate(raw, encoding)
47
47
  case encoding
48
- when 'gzip' then uncompress_gzip(raw)
49
- when 'deflate' then inflate_deflate(raw)
48
+ when 'gzip'
49
+ return raw unless raw.start_with?(GZIP_MAGIC)
50
+
51
+ uncompress_gzip(raw)
50
52
  when 'br' then try_brotli(raw, require_html: false)
53
+ when 'deflate' then inflate_deflate(raw)
51
54
  end
52
55
  rescue Zlib::Error, Brotli::Error, ArgumentError
53
56
  nil
@@ -55,12 +58,6 @@ module Html2rss
55
58
  module_function :inflate
56
59
  private_class_method :inflate
57
60
 
58
- def uncompress_gzip(raw)
59
- Zlib::GzipReader.wrap(StringIO.new(raw), encoding: 'ASCII-8BIT', &:read)
60
- end
61
- module_function :uncompress_gzip
62
- private_class_method :uncompress_gzip
63
-
64
61
  def inflate_deflate(raw)
65
62
  inflater = nil
66
63
  Zlib::Inflate.inflate(raw)
@@ -73,6 +70,14 @@ module Html2rss
73
70
  module_function :inflate_deflate
74
71
  private_class_method :inflate_deflate
75
72
 
73
+ def uncompress_gzip(raw)
74
+ Zlib::GzipReader.wrap(StringIO.new(raw), encoding: 'ASCII-8BIT', &:read)
75
+ rescue Zlib::Error
76
+ nil
77
+ end
78
+ module_function :uncompress_gzip
79
+ private_class_method :uncompress_gzip
80
+
76
81
  def try_brotli(raw, require_html: true)
77
82
  inflated = Brotli.inflate(raw)
78
83
  return inflated unless require_html
@@ -0,0 +1,228 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'httpx'
4
+ require 'httpx/plugins/follow_redirects'
5
+ require 'httpx/plugins/callbacks'
6
+ require 'httpx/plugins/ssrf_filter'
7
+ require 'httpx/plugins/brotli'
8
+ require 'httpx/plugins/fiber_concurrency'
9
+ require 'httpx/plugins/retries'
10
+
11
+ module Html2rss
12
+ class RequestService
13
+ ##
14
+ # Strategy to use HTTPX for HTTP requests.
15
+ # Provides native HTTP/2 with ALPN, built-in SSRF protection, streaming byte limits,
16
+ # and redirect handling without monkey-patching.
17
+ # rubocop:disable-next Metrics/ClassLength -- terminal redirect retry colocated with HTTPX transport
18
+ class HttpxStrategy < Strategy
19
+ class << self
20
+ # rubocop:disable ThreadSafety/ClassInstanceVariable
21
+ # @return [HTTPX::Session]
22
+ def base_session
23
+ @base_sessions ||= {}
24
+ @base_sessions[HTTPX::Session] ||= HTTPX
25
+ .plugin(:follow_redirects)
26
+ .plugin(:callbacks)
27
+ .plugin(:brotli)
28
+ .plugin(:fiber_concurrency)
29
+ .plugin(:retries)
30
+ end
31
+
32
+ # @return [HTTPX::Session]
33
+ def base_ssrf_session
34
+ @base_ssrf_sessions ||= {}
35
+ @base_ssrf_sessions[HTTPX::Session] ||= base_session.plugin(:ssrf_filter)
36
+ end
37
+ # rubocop:enable ThreadSafety/ClassInstanceVariable
38
+ end
39
+
40
+ ##
41
+ # @return [ResponseGuard]
42
+ attr_reader :response_guard
43
+
44
+ # Executes the request with runtime policy enforcement, returning the normalized response.
45
+ #
46
+ # @return [Response] normalized response
47
+ def perform_execute
48
+ deadline = request_deadline
49
+ @response_guard = ResponseGuard.new(policy: ctx.policy)
50
+ reset_redirect_tracking!
51
+ raw_response = request_with_terminal_redirect_retry(response_guard, deadline:)
52
+ build_response(raw_response)
53
+ end
54
+
55
+ private
56
+
57
+ def request_deadline
58
+ monotonic_now + ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
59
+ end
60
+
61
+ def reset_redirect_tracking!
62
+ @last_redirect_to = nil
63
+ @terminal_redirect_retried = false
64
+ @request_url_override = nil
65
+ @current_url = request_url
66
+ end
67
+
68
+ def request_with_terminal_redirect_retry(response_guard, deadline:)
69
+ raw_response = execute_http_request(response_guard, deadline:)
70
+ return raw_response unless redirect_limit_reached?(raw_response)
71
+
72
+ unless terminal_redirect_retryable?(raw_response)
73
+ raise RedirectLimitReached, "Too many redirects (status #{raw_response.status})"
74
+ end
75
+
76
+ retry_from_terminal_redirect!(raw_response, response_guard, deadline:)
77
+ end
78
+
79
+ def redirect_limit_reached?(response)
80
+ return false unless response.is_a?(HTTPX::Response)
81
+ return false if response.status == 304
82
+ return false unless (300..399).cover?(response.status)
83
+
84
+ location = response.headers['location']
85
+ !location.nil? && !location.strip.empty?
86
+ end
87
+
88
+ def terminal_redirect_url(raw_response)
89
+ location = raw_response.headers['location']
90
+ if location && !location.empty?
91
+ base = raw_response.uri || request_url
92
+ return normalize_url(URI.join(base.to_s, location))
93
+ end
94
+
95
+ @last_redirect_to
96
+ end
97
+
98
+ def terminal_redirect_retryable?(raw_response)
99
+ return false if @terminal_redirect_retried
100
+
101
+ target = terminal_redirect_url(raw_response)
102
+ target && target.to_s != request_url.to_s
103
+ end
104
+
105
+ def retry_from_terminal_redirect!(raw_response, response_guard, deadline:)
106
+ terminal_url = terminal_redirect_url(raw_response)
107
+ @terminal_redirect_retried = true
108
+ Log.debug("#{self.class}: redirect limit reached; retrying once from #{terminal_url}")
109
+ begin_terminal_url_request!(terminal_url)
110
+ new_response = execute_http_request(response_guard, deadline:, consume_budget: false)
111
+ if redirect_limit_reached?(new_response)
112
+ raise RedirectLimitReached, "Too many redirects (status #{new_response.status})"
113
+ end
114
+
115
+ new_response
116
+ end
117
+
118
+ def begin_terminal_url_request!(terminal_url)
119
+ ctx.policy.validate_request!(url: terminal_url, origin_url: ctx.origin_url, relation: ctx.relation)
120
+ @request_url_override = terminal_url
121
+ @last_redirect_to = nil
122
+ @current_url = terminal_url
123
+ end
124
+
125
+ def request_url
126
+ @request_url_override || ctx.url
127
+ end
128
+
129
+ def execute_http_request(response_guard, deadline:, consume_budget: true)
130
+ preflight!(consume_budget:)
131
+ session = build_session(response_guard, deadline:)
132
+ response = session.get(request_url.to_s, headers: ctx.headers)
133
+ raise response.error if response.is_a?(HTTPX::ErrorResponse)
134
+
135
+ response
136
+ end
137
+
138
+ def build_session(_response_guard, deadline:)
139
+ session_client(deadline)
140
+ end
141
+
142
+ def session_client(deadline)
143
+ session = ctx.policy.allow_private_networks? ? self.class.base_session : self.class.base_ssrf_session
144
+ session.with(session_options(deadline))
145
+ end
146
+
147
+ def session_options(deadline)
148
+ remaining = remaining_timeout_seconds(deadline)
149
+ {
150
+ timeout: session_timeouts(remaining),
151
+ max_redirects: ctx.policy.max_redirects,
152
+ follow_insecure_redirects: false,
153
+ max_response_body_size: ctx.policy.max_response_bytes,
154
+ resolver_class: :system,
155
+ redirect_on: redirect_callback
156
+ }
157
+ end
158
+
159
+ def session_timeouts(remaining)
160
+ {
161
+ connect_timeout: [ctx.policy.connect_timeout_seconds, remaining].min,
162
+ read_timeout: [ctx.policy.read_timeout_seconds, remaining].min,
163
+ operation_timeout: remaining,
164
+ request_timeout: remaining,
165
+ total_request_timeout: remaining
166
+ }
167
+ end
168
+
169
+ def redirect_callback
170
+ lambda do |redirect_uri|
171
+ to_url = normalize_url(redirect_uri)
172
+ from_url = @current_url
173
+ @last_redirect_to = to_url
174
+ ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
175
+ @current_url = to_url
176
+ true
177
+ end
178
+ end
179
+
180
+ def build_response(response)
181
+ headers = response.headers.to_h
182
+ Response.new(
183
+ body: CompressedBody.decode(response.body.to_s, headers:),
184
+ headers:,
185
+ url: response_url(response),
186
+ status: response.status
187
+ )
188
+ end
189
+
190
+ def response_url(response)
191
+ return ctx.url unless (uri = response.uri)
192
+
193
+ Html2rss::Url.from_absolute(uri.to_s)
194
+ end
195
+
196
+ def normalize_url(url)
197
+ Html2rss::Url.from_absolute(url.to_s)
198
+ end
199
+
200
+ def remaining_timeout_seconds(deadline)
201
+ remaining = deadline - monotonic_now
202
+ raise RequestTimedOut, 'Request timed out' if remaining <= 0
203
+
204
+ remaining
205
+ end
206
+
207
+ def monotonic_now
208
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
209
+ end
210
+
211
+ def timeout_error?(error)
212
+ error.is_a?(HTTPX::TimeoutError) || super
213
+ end
214
+
215
+ def ssrf_error?(error)
216
+ error.is_a?(HTTPX::ServerSideRequestForgeryError) || super
217
+ end
218
+
219
+ def response_too_large_error?(error)
220
+ (error.is_a?(HTTPX::Error) && error.message.include?('maximum response body size exceeded')) || super
221
+ end
222
+
223
+ def connection_error?(error)
224
+ error.is_a?(HTTPX::ConnectionError) || error.is_a?(HTTPX::TLSError) || super
225
+ end
226
+ end
227
+ end
228
+ end
@@ -84,11 +84,10 @@ module Html2rss
84
84
  # Returns the default request policy.
85
85
  #
86
86
  # @return [Policy] a default, frozen policy instance
87
- # rubocop:disable Layout/ClassStructure
87
+ # rubocop:disable-next Layout/ClassStructure
88
88
  def self.default
89
89
  new
90
90
  end
91
- # rubocop:enable Layout/ClassStructure
92
91
 
93
92
  ##
94
93
  # Validates whether a request target is permitted for the given context.
@@ -31,7 +31,7 @@ module Html2rss
31
31
  # @param status [Integer, nil] the HTTP status code when available
32
32
  # @param transport_meta [Hash] allowlisted upstream telemetry (frozen when present)
33
33
  # @param captured_responses [Array<Hash>] JSON XHR/fetch bodies captured during browser scrapes
34
- # rubocop:disable Metrics/MethodLength, Metrics/ParameterLists -- transport + capture fields stay co-located
34
+ # rubocop:disable-next Metrics/MethodLength, Metrics/ParameterLists -- transport + capture fields stay co-located
35
35
  def initialize(body:, url:, headers: {}, status: nil, transport_meta: EMPTY_TRANSPORT_META,
36
36
  captured_responses: EMPTY_CAPTURED_RESPONSES)
37
37
  @body = body
@@ -50,7 +50,6 @@ module Html2rss
50
50
  captured_responses.freeze
51
51
  end
52
52
  end
53
- # rubocop:enable Metrics/MethodLength, Metrics/ParameterLists
54
53
 
55
54
  # @return [String] the raw body of the response
56
55
  attr_reader :body
@@ -160,12 +159,12 @@ module Html2rss
160
159
  return false if ct.empty? || ct.include?('html') || ct.include?('json')
161
160
 
162
161
  # Substring match against Content-Type (not Array#intersect? on a String).
163
- FEED_CT_MARKERS.any? { |marker| ct.include?(marker) } # rubocop:disable Style/ArrayIntersect
162
+ FEED_CT_MARKERS.any? { |marker| ct.include?(marker) }
164
163
  end
165
164
 
166
165
  def feed_looking_body?
167
166
  snippet = body.to_s[0, 800].downcase
168
- FEED_BODY_MARKERS.any? { |marker| snippet.include?(marker) } # rubocop:disable Style/ArrayIntersect
167
+ FEED_BODY_MARKERS.any? { |marker| snippet.include?(marker) }
169
168
  end
170
169
  end
171
170
  end
@@ -9,22 +9,6 @@ module Html2rss
9
9
  # @param policy [Policy] request policy that defines byte ceilings
10
10
  def initialize(policy:)
11
11
  @policy = policy
12
- @streamed_bytes = 0
13
- end
14
-
15
- ##
16
- # Validates response headers and streamed byte count.
17
- #
18
- # @param total_bytes [Integer] cumulative byte count received so far
19
- # @param headers [Hash, nil] response headers if known
20
- # @return [void]
21
- # @raise [ResponseTooLarge] if the response exceeds configured limits
22
- def inspect_chunk!(total_bytes:, headers: nil)
23
- header_length = headers&.fetch('content-length', headers&.fetch('Content-Length', nil))
24
- raise_if_too_large!(header_length.to_i, policy.max_response_bytes) if header_length
25
-
26
- @streamed_bytes = total_bytes
27
- raise_if_too_large!(@streamed_bytes, policy.max_response_bytes)
28
12
  end
29
13
 
30
14
  ##
@@ -97,11 +97,16 @@ module Html2rss
97
97
  # @param error [StandardError]
98
98
  # @return [void]
99
99
  # @raise [StandardError]
100
+ # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- error translation dispatch
100
101
  def handle_error(error)
101
102
  if timeout_error?(error)
102
103
  log_timeout!(reason: 'transport')
103
104
  Log.debug("#{self.class}: transport timeout message=#{error.message}")
104
105
  raise RequestTimedOut, error.message
106
+ elsif ssrf_error?(error)
107
+ raise PrivateNetworkDenied, error.message
108
+ elsif response_too_large_error?(error)
109
+ raise ResponseTooLarge, "Response exceeded #{ctx.policy.max_response_bytes} bytes"
105
110
  elsif connection_error?(error)
106
111
  translate_connection_error(error)
107
112
  else
@@ -111,7 +116,7 @@ module Html2rss
111
116
 
112
117
  # @param reason [String] timeout classification (budget_exhausted / transport)
113
118
  # @return [void]
114
- # rubocop:disable Metrics/AbcSize -- structured timeout fields stay in one log line
119
+ # rubocop:disable-next Metrics/AbcSize -- structured timeout fields stay in one log line
115
120
  def log_timeout!(reason:)
116
121
  remaining = ctx.budget.remaining_timeout_seconds
117
122
  remaining_label = remaining.nil? ? 'untracked' : format('%.3f', remaining)
@@ -124,7 +129,6 @@ module Html2rss
124
129
  ]
125
130
  Log.info("#{self.class}: request timeout #{detail.join(' ')}")
126
131
  end
127
- # rubocop:enable Metrics/AbcSize
128
132
 
129
133
  # @param error [StandardError]
130
134
  # @return [void]
@@ -136,14 +140,25 @@ module Html2rss
136
140
  # @param error [StandardError]
137
141
  # @return [Boolean]
138
142
  def timeout_error?(error)
139
- error.is_a?(Faraday::TimeoutError) ||
140
- error.is_a?(Timeout::Error)
143
+ error.is_a?(Timeout::Error)
144
+ end
145
+
146
+ # @param _error [StandardError]
147
+ # @return [Boolean]
148
+ def ssrf_error?(_error)
149
+ false
150
+ end
151
+
152
+ # @param _error [StandardError]
153
+ # @return [Boolean]
154
+ def response_too_large_error?(_error)
155
+ false
141
156
  end
142
157
 
143
158
  # @param error [StandardError]
144
159
  # @return [Boolean]
145
160
  def connection_error?(error)
146
- error.is_a?(Faraday::ConnectionFailed) || error.is_a?(Faraday::SSLError)
161
+ error.is_a?(SocketError) || error.is_a?(SystemCallError)
147
162
  end
148
163
  end
149
164
  end
@@ -1,18 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'singleton'
4
- require 'forwardable'
5
-
6
3
  module Html2rss
7
4
  ##
8
5
  # Requests website URLs to retrieve their HTML for further processing.
9
- # Provides concrete transport strategies (e.g. Faraday, Botasaurus).
6
+ # Provides concrete transport strategies (e.g. HttpxStrategy, BotasaurusStrategy).
10
7
  #
11
8
  # Feed-level +:auto+ is not registered here — {FeedPipeline::StrategyPlan} resolves
12
9
  # it to a concrete strategy (or {FeedPipeline::AutoFallback} chain) before execute.
13
10
  class RequestService
14
- include Singleton
15
-
16
11
  # Raised when an unknown request strategy is requested.
17
12
  class UnknownStrategy < Html2rss::Error; end
18
13
  # Raised when a URL cannot be parsed or validated.
@@ -29,6 +24,8 @@ module Html2rss
29
24
  class CrossOriginFollowUpDenied < Html2rss::Error; end
30
25
  # Raised when a response exceeds configured size limits.
31
26
  class ResponseTooLarge < Html2rss::Error; end
27
+ # Raised when HTTP redirect limits are exceeded.
28
+ class RedirectLimitReached < Html2rss::Error; end
32
29
  # Raised when blocked content surfaces are detected.
33
30
  class BlockedSurfaceDetected < Html2rss::Error; end
34
31
 
@@ -57,94 +54,60 @@ module Html2rss
57
54
  # Raised when Botasaurus responds but the scrape fails (upstream error, bad payload).
58
55
  class BotasaurusServiceError < Html2rss::Error; end
59
56
 
60
- class << self
61
- extend Forwardable
62
-
63
- %i[default_strategy_name
64
- default_strategy_name=
65
- strategy_names
66
- register_strategy
67
- unregister_strategy
68
- strategy_registered?
69
- execute].each do |method|
70
- def_delegator :instance, method
71
- end
72
- end
73
-
74
- def initialize
75
- @strategies = {
76
- faraday: FaradayStrategy,
77
- botasaurus: BotasaurusStrategy,
78
- local_file: LocalFileStrategy
79
- }
80
- @default_strategy_name = :faraday
81
- end
57
+ # Map of supported strategy names to their implementation classes.
58
+ # @return [Hash{Symbol => Class<Strategy>}]
59
+ STRATEGIES = {
60
+ default: HttpxStrategy,
61
+ httpx: HttpxStrategy,
62
+ faraday: HttpxStrategy,
63
+ botasaurus: BotasaurusStrategy,
64
+ local_file: LocalFileStrategy
65
+ }.freeze
82
66
 
83
- # @return [Symbol] the default strategy name
84
- attr_reader :default_strategy_name
67
+ # Canonical default strategy symbol.
68
+ # @return [Symbol]
69
+ DEFAULT_STRATEGY_NAME = :default
85
70
 
86
- ##
87
- # Sets the default strategy.
88
- # @param strategy [Symbol] the name of the strategy
89
- # @return [Symbol] the selected default strategy name
90
- # @raise [UnknownStrategy] if the strategy is not registered
91
- def default_strategy_name=(strategy)
92
- raise UnknownStrategy unless strategy_registered?(strategy)
93
-
94
- @default_strategy_name = strategy.to_sym
95
- end
71
+ class << self
72
+ # @return [Symbol] the default strategy name
73
+ def default_strategy_name = DEFAULT_STRATEGY_NAME
96
74
 
97
- # @return [Array<String>] the names of the registered strategies
98
- def strategy_names = @strategies.keys.map(&:to_s)
75
+ # @return [Array<String>] the names of the registered strategies
76
+ def strategy_names = STRATEGIES.keys.map(&:to_s)
99
77
 
100
- ##
101
- # Registers a new strategy.
102
- # @param name [Symbol] the name of the strategy
103
- # @param strategy_class [Class] the class implementing the strategy
104
- # @return [Class] the registered strategy class
105
- # @raise [ArgumentError] if strategy_class is not a Class
106
- def register_strategy(name, strategy_class)
107
- unless strategy_class.is_a?(Class)
108
- raise ArgumentError, "Expected a Class for strategy, got #{strategy_class.class}"
78
+ ##
79
+ # Checks if a strategy is registered.
80
+ # @param name [Symbol, String] the name of the strategy
81
+ # @return [Boolean] true if the strategy is registered, false otherwise.
82
+ def strategy_registered?(name)
83
+ STRATEGIES.key?(name.to_sym)
109
84
  end
110
85
 
111
- @strategies[name.to_sym] = strategy_class
112
- end
113
-
114
- ##
115
- # Checks if a strategy is registered.
116
- # @param name [Symbol] the name of the strategy
117
- # @return [Boolean] true if the strategy is registered, false otherwise.
118
- def strategy_registered?(name)
119
- @strategies.key?(name.to_sym)
120
- end
121
-
122
- ##
123
- # Unregisters a strategy.
124
- # @param name [Symbol] the name of the strategy
125
- # @return [Boolean] true if the strategy was unregistered, false otherwise.
126
- # @raise [ArgumentError] if attempting to unregister the default strategy.
127
- def unregister_strategy(name) # rubocop:disable Naming/PredicateMethod
128
- name_sym = name.to_sym
129
- raise ArgumentError, 'Cannot unregister the default strategy.' if name_sym == @default_strategy_name
86
+ ##
87
+ # Executes the request using the specified strategy.
88
+ # @param ctx [Context] the context for the request.
89
+ # @param strategy [Symbol, String] the strategy to use (defaults to the default strategy).
90
+ # @return [Response] the response from the executed strategy.
91
+ # @raise [UnknownStrategy] if the strategy is not registered.
92
+ def execute(ctx, strategy: default_strategy_name)
93
+ strategy_sym = strategy.to_sym
94
+ strategy_class = STRATEGIES.fetch(strategy_sym) do
95
+ raise UnknownStrategy,
96
+ "The strategy '#{strategy}' is not known. Available strategies: #{strategy_names.join(', ')}"
97
+ end
98
+
99
+ warn_migration_strategy(strategy_sym) if strategy_sym == :faraday
100
+ strategy_class.new(ctx).execute
101
+ end
130
102
 
131
- !!@strategies.delete(name_sym)
132
- end
103
+ private
133
104
 
134
- ##
135
- # Executes the request using the specified strategy.
136
- # @param ctx [Context] the context for the request.
137
- # @param strategy [Symbol] the strategy to use (defaults to the default strategy).
138
- # @return [Response] the response from the executed strategy.
139
- # @raise [ArgumentError] if the context is nil.
140
- # @raise [UnknownStrategy] if the strategy is not registered.
141
- def execute(ctx, strategy: default_strategy_name)
142
- strategy_class = @strategies.fetch(strategy.to_sym) do
143
- raise UnknownStrategy,
144
- "The strategy '#{strategy}' is not known. Available strategies: #{strategy_names.join(', ')}"
105
+ def warn_migration_strategy(strategy)
106
+ message = "RequestService: strategy ':#{strategy}' is deprecated for migration and will be removed " \
107
+ "in a future release. Use ':default' instead."
108
+ warn(message, category: :deprecated)
109
+ Log.warn(message)
145
110
  end
146
-
147
- strategy_class.new(ctx).execute
148
111
  end
149
112
  end
150
113
  end