html2rss 0.28.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +84 -27
- data/html2rss.gemspec +1 -4
- data/lib/html2rss/article.rb +1 -2
- data/lib/html2rss/auto_source/README.md +8 -8
- data/lib/html2rss/auto_source/cleanup.rb +92 -1
- data/lib/html2rss/auto_source/scraper/json_state/document_scanner.rb +3 -5
- data/lib/html2rss/auto_source/scraper/meta_oembed.rb +15 -5
- data/lib/html2rss/auto_source/scraper/microdata.rb +8 -6
- data/lib/html2rss/auto_source/scraper/native_feed.rb +5 -2
- data/lib/html2rss/auto_source/scraper/schema.rb +39 -21
- data/lib/html2rss/auto_source/scraper.rb +2 -3
- data/lib/html2rss/auto_source/segmenter/cluster.rb +2 -3
- data/lib/html2rss/auto_source/segmenter/primary_link.rb +1 -2
- data/lib/html2rss/auto_source/segmenter.rb +1 -2
- data/lib/html2rss/auto_source.rb +25 -11
- data/lib/html2rss/batch.rb +169 -0
- data/lib/html2rss/capture/README.md +15 -6
- data/lib/html2rss/capture.rb +159 -16
- data/lib/html2rss/channel.rb +1 -2
- data/lib/html2rss/cli/probe_view.rb +66 -0
- data/lib/html2rss/cli/render.rb +124 -0
- data/lib/html2rss/cli/validate.rb +91 -0
- data/lib/html2rss/cli.rb +318 -193
- data/lib/html2rss/config/request_headers.rb +9 -16
- data/lib/html2rss/config/schema.rb +7 -5
- data/lib/html2rss/config/validator.rb +1 -0
- data/lib/html2rss/config.rb +68 -19
- data/lib/html2rss/doctor/botasaurus.rb +138 -0
- data/lib/html2rss/feed_builder/rss.rb +1 -2
- data/lib/html2rss/feed_pipeline/README.md +11 -11
- data/lib/html2rss/feed_pipeline/auto_fallback.rb +30 -11
- data/lib/html2rss/feed_pipeline/strategy_plan.rb +1 -1
- data/lib/html2rss/feed_pipeline.rb +27 -16
- data/lib/html2rss/feed_resolution/README.md +11 -11
- data/lib/html2rss/feed_resolution.rb +6 -12
- data/lib/html2rss/html/article_extractor/heading_extractor.rb +24 -14
- data/lib/html2rss/html/article_extractor.rb +4 -9
- data/lib/html2rss/html/feed_link.rb +4 -2
- data/lib/html2rss/html/navigator.rb +4 -39
- data/lib/html2rss/html/probe.rb +82 -0
- data/lib/html2rss/html/sst_article_extractor.rb +1 -2
- data/lib/html2rss/link_destination/noise_policy.rb +1 -2
- data/lib/html2rss/link_destination/path_classifier.rb +20 -19
- data/lib/html2rss/mcp/README.md +80 -0
- data/lib/html2rss/mcp/contract.rb +144 -18
- data/lib/html2rss/mcp/outcome/playbook.rb +122 -0
- data/lib/html2rss/mcp/outcome.rb +112 -42
- data/lib/html2rss/mcp/runtime.rb +45 -0
- data/lib/html2rss/mcp/server/tools.rb +277 -0
- data/lib/html2rss/mcp/server.rb +92 -233
- data/lib/html2rss/mcp.rb +3 -0
- data/lib/html2rss/page_recon/README.md +69 -0
- data/lib/html2rss/page_recon/diagnostics.rb +211 -0
- data/lib/html2rss/page_recon.rb +56 -2
- data/lib/html2rss/recon.rb +252 -0
- data/lib/html2rss/request_service/blocked_surface.rb +1 -0
- data/lib/html2rss/request_service/botasaurus_strategy.rb +32 -28
- data/lib/html2rss/request_service/compressed_body.rb +13 -8
- data/lib/html2rss/request_service/httpx_strategy.rb +228 -0
- data/lib/html2rss/request_service/policy.rb +1 -2
- data/lib/html2rss/request_service/response.rb +3 -4
- data/lib/html2rss/request_service/response_guard.rb +0 -16
- data/lib/html2rss/request_service/strategy.rb +20 -5
- data/lib/html2rss/request_service.rb +48 -85
- data/lib/html2rss/request_session.rb +1 -2
- data/lib/html2rss/scrape_target.rb +1 -1
- data/lib/html2rss/selectors/post_processors/sanitize_html.rb +2 -4
- data/lib/html2rss/selectors/post_processors.rb +0 -1
- data/lib/html2rss/selectors.rb +33 -10
- data/lib/html2rss/sst/attrs.rb +1 -2
- data/lib/html2rss/sst/normalizer.rb +49 -11
- data/lib/html2rss/status.rb +3 -6
- data/lib/html2rss/syndication/README.md +6 -6
- data/lib/html2rss/syndication/discovery.rb +2 -4
- data/lib/html2rss/test/enhance_audit.rb +192 -0
- data/lib/html2rss/test.rb +459 -0
- data/lib/html2rss/url.rb +1 -2
- data/lib/html2rss/version.rb +1 -1
- data/lib/html2rss.rb +157 -35
- data/schema/html2rss-config.schema.json +9 -29
- metadata +21 -56
- data/lib/html2rss/mcp/inspect.rb +0 -138
- data/lib/html2rss/request_service/faraday_strategy.rb +0 -229
- data/lib/html2rss/selectors/post_processors/html_to_markdown.rb +0 -59
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require '
|
|
3
|
+
require 'httpx'
|
|
4
4
|
require 'json'
|
|
5
5
|
require 'securerandom'
|
|
6
6
|
|
|
@@ -11,7 +11,7 @@ module Html2rss
|
|
|
11
11
|
class BotasaurusStrategy < Strategy
|
|
12
12
|
# Content-Type negotiation for the scrape API transport hop.
|
|
13
13
|
TRANSPORT_ACCEPT = 'application/json'
|
|
14
|
-
# Disable compressed bodies so
|
|
14
|
+
# Disable compressed bodies so HTTPX returns raw JSON without implicit decoding surprises.
|
|
15
15
|
TRANSPORT_ENCODING = 'identity'
|
|
16
16
|
# Correlates each POST /scrape with botasaurus-scrape-api request logs.
|
|
17
17
|
REQUEST_ID_HEADER = 'X-Request-Id'
|
|
@@ -28,10 +28,15 @@ module Html2rss
|
|
|
28
28
|
def post_scrape_request
|
|
29
29
|
request_id = SecureRandom.uuid
|
|
30
30
|
Log.debug("#{self.class}: POST /scrape #{REQUEST_ID_HEADER}=#{request_id}")
|
|
31
|
-
|
|
32
|
-
|
|
31
|
+
payload = JSON.generate(contract.request_payload)
|
|
32
|
+
response = client.post(scrape_endpoint, body: payload, headers: post_headers(request_id))
|
|
33
|
+
raise response.error if response.is_a?(HTTPX::ErrorResponse)
|
|
34
|
+
|
|
35
|
+
contract.parse_response(response)
|
|
33
36
|
end
|
|
34
37
|
|
|
38
|
+
def scrape_endpoint = "#{scraper_base_url.to_s.chomp('/')}/scrape"
|
|
39
|
+
|
|
35
40
|
def build_response(parsed_response)
|
|
36
41
|
Response.new(
|
|
37
42
|
body: parsed_response.html,
|
|
@@ -44,22 +49,14 @@ module Html2rss
|
|
|
44
49
|
end
|
|
45
50
|
|
|
46
51
|
def raise_from_error!(error)
|
|
47
|
-
|
|
48
|
-
raise_if_timed_out!(error)
|
|
49
|
-
raise BotasaurusServiceError, error.failure_message
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
def raise_if_challenge_blocked!(error)
|
|
53
|
-
return unless error.challenge_block?
|
|
52
|
+
raise BlockedSurfaceDetected, "Blocked surface detected: #{error.challenge_message}" if error.challenge_block?
|
|
54
53
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
return unless error.timeout?
|
|
54
|
+
if error.timeout?
|
|
55
|
+
log_timeout!(reason: 'botasaurus_upstream')
|
|
56
|
+
raise RequestTimedOut.new(error.failure_message, timeout_phase: error.timeout_phase)
|
|
57
|
+
end
|
|
60
58
|
|
|
61
|
-
|
|
62
|
-
raise RequestTimedOut.new(error.failure_message, timeout_phase: error.timeout_phase)
|
|
59
|
+
raise BotasaurusServiceError, error.failure_message
|
|
63
60
|
end
|
|
64
61
|
|
|
65
62
|
def response_url(final_url)
|
|
@@ -79,8 +76,18 @@ module Html2rss
|
|
|
79
76
|
end
|
|
80
77
|
|
|
81
78
|
def client
|
|
82
|
-
|
|
83
|
-
@client ||=
|
|
79
|
+
timeout = attempt_timeout_seconds.to_f
|
|
80
|
+
@client ||= HTTPX.with(headers: client_headers, timeout: client_timeouts(timeout), resolver_class: :system)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def client_timeouts(timeout)
|
|
84
|
+
{
|
|
85
|
+
operation_timeout: timeout,
|
|
86
|
+
connect_timeout: [ctx.policy.connect_timeout_seconds, timeout].min,
|
|
87
|
+
read_timeout: [ctx.policy.read_timeout_seconds, timeout].min,
|
|
88
|
+
request_timeout: timeout,
|
|
89
|
+
total_request_timeout: timeout
|
|
90
|
+
}
|
|
84
91
|
end
|
|
85
92
|
|
|
86
93
|
def client_headers
|
|
@@ -92,14 +99,7 @@ module Html2rss
|
|
|
92
99
|
end
|
|
93
100
|
|
|
94
101
|
def post_headers(request_id)
|
|
95
|
-
{
|
|
96
|
-
'Content-Type' => 'application/json',
|
|
97
|
-
REQUEST_ID_HEADER => request_id
|
|
98
|
-
}
|
|
99
|
-
end
|
|
100
|
-
|
|
101
|
-
def request_options
|
|
102
|
-
{ timeout: attempt_timeout_seconds.to_i }
|
|
102
|
+
{ 'Content-Type' => 'application/json', REQUEST_ID_HEADER => request_id }
|
|
103
103
|
end
|
|
104
104
|
|
|
105
105
|
def attempt_timeout_seconds
|
|
@@ -123,6 +123,10 @@ module Html2rss
|
|
|
123
123
|
def translate_connection_error(error)
|
|
124
124
|
raise BotasaurusConnectionFailed, "Botasaurus connection failed: #{error.message}"
|
|
125
125
|
end
|
|
126
|
+
|
|
127
|
+
def timeout_error?(error) = error.is_a?(HTTPX::TimeoutError) || super
|
|
128
|
+
|
|
129
|
+
def connection_error?(error) = error.is_a?(HTTPX::ConnectionError) || error.is_a?(HTTPX::TLSError) || super
|
|
126
130
|
end
|
|
127
131
|
end
|
|
128
132
|
end
|
|
@@ -45,9 +45,12 @@ module Html2rss
|
|
|
45
45
|
|
|
46
46
|
def inflate(raw, encoding)
|
|
47
47
|
case encoding
|
|
48
|
-
when 'gzip'
|
|
49
|
-
|
|
48
|
+
when 'gzip'
|
|
49
|
+
return raw unless raw.start_with?(GZIP_MAGIC)
|
|
50
|
+
|
|
51
|
+
uncompress_gzip(raw)
|
|
50
52
|
when 'br' then try_brotli(raw, require_html: false)
|
|
53
|
+
when 'deflate' then inflate_deflate(raw)
|
|
51
54
|
end
|
|
52
55
|
rescue Zlib::Error, Brotli::Error, ArgumentError
|
|
53
56
|
nil
|
|
@@ -55,12 +58,6 @@ module Html2rss
|
|
|
55
58
|
module_function :inflate
|
|
56
59
|
private_class_method :inflate
|
|
57
60
|
|
|
58
|
-
def uncompress_gzip(raw)
|
|
59
|
-
Zlib::GzipReader.wrap(StringIO.new(raw), encoding: 'ASCII-8BIT', &:read)
|
|
60
|
-
end
|
|
61
|
-
module_function :uncompress_gzip
|
|
62
|
-
private_class_method :uncompress_gzip
|
|
63
|
-
|
|
64
61
|
def inflate_deflate(raw)
|
|
65
62
|
inflater = nil
|
|
66
63
|
Zlib::Inflate.inflate(raw)
|
|
@@ -73,6 +70,14 @@ module Html2rss
|
|
|
73
70
|
module_function :inflate_deflate
|
|
74
71
|
private_class_method :inflate_deflate
|
|
75
72
|
|
|
73
|
+
def uncompress_gzip(raw)
|
|
74
|
+
Zlib::GzipReader.wrap(StringIO.new(raw), encoding: 'ASCII-8BIT', &:read)
|
|
75
|
+
rescue Zlib::Error
|
|
76
|
+
nil
|
|
77
|
+
end
|
|
78
|
+
module_function :uncompress_gzip
|
|
79
|
+
private_class_method :uncompress_gzip
|
|
80
|
+
|
|
76
81
|
def try_brotli(raw, require_html: true)
|
|
77
82
|
inflated = Brotli.inflate(raw)
|
|
78
83
|
return inflated unless require_html
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'httpx'
|
|
4
|
+
require 'httpx/plugins/follow_redirects'
|
|
5
|
+
require 'httpx/plugins/callbacks'
|
|
6
|
+
require 'httpx/plugins/ssrf_filter'
|
|
7
|
+
require 'httpx/plugins/brotli'
|
|
8
|
+
require 'httpx/plugins/fiber_concurrency'
|
|
9
|
+
require 'httpx/plugins/retries'
|
|
10
|
+
|
|
11
|
+
module Html2rss
|
|
12
|
+
class RequestService
|
|
13
|
+
##
|
|
14
|
+
# Strategy to use HTTPX for HTTP requests.
|
|
15
|
+
# Provides native HTTP/2 with ALPN, built-in SSRF protection, streaming byte limits,
|
|
16
|
+
# and redirect handling without monkey-patching.
|
|
17
|
+
# rubocop:disable-next Metrics/ClassLength -- terminal redirect retry colocated with HTTPX transport
|
|
18
|
+
class HttpxStrategy < Strategy
|
|
19
|
+
class << self
|
|
20
|
+
# rubocop:disable ThreadSafety/ClassInstanceVariable
|
|
21
|
+
# @return [HTTPX::Session]
|
|
22
|
+
def base_session
|
|
23
|
+
@base_sessions ||= {}
|
|
24
|
+
@base_sessions[HTTPX::Session] ||= HTTPX
|
|
25
|
+
.plugin(:follow_redirects)
|
|
26
|
+
.plugin(:callbacks)
|
|
27
|
+
.plugin(:brotli)
|
|
28
|
+
.plugin(:fiber_concurrency)
|
|
29
|
+
.plugin(:retries)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# @return [HTTPX::Session]
|
|
33
|
+
def base_ssrf_session
|
|
34
|
+
@base_ssrf_sessions ||= {}
|
|
35
|
+
@base_ssrf_sessions[HTTPX::Session] ||= base_session.plugin(:ssrf_filter)
|
|
36
|
+
end
|
|
37
|
+
# rubocop:enable ThreadSafety/ClassInstanceVariable
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
##
|
|
41
|
+
# @return [ResponseGuard]
|
|
42
|
+
attr_reader :response_guard
|
|
43
|
+
|
|
44
|
+
# Executes the request with runtime policy enforcement, returning the normalized response.
|
|
45
|
+
#
|
|
46
|
+
# @return [Response] normalized response
|
|
47
|
+
def perform_execute
|
|
48
|
+
deadline = request_deadline
|
|
49
|
+
@response_guard = ResponseGuard.new(policy: ctx.policy)
|
|
50
|
+
reset_redirect_tracking!
|
|
51
|
+
raw_response = request_with_terminal_redirect_retry(response_guard, deadline:)
|
|
52
|
+
build_response(raw_response)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
def request_deadline
|
|
58
|
+
monotonic_now + ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def reset_redirect_tracking!
|
|
62
|
+
@last_redirect_to = nil
|
|
63
|
+
@terminal_redirect_retried = false
|
|
64
|
+
@request_url_override = nil
|
|
65
|
+
@current_url = request_url
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def request_with_terminal_redirect_retry(response_guard, deadline:)
|
|
69
|
+
raw_response = execute_http_request(response_guard, deadline:)
|
|
70
|
+
return raw_response unless redirect_limit_reached?(raw_response)
|
|
71
|
+
|
|
72
|
+
unless terminal_redirect_retryable?(raw_response)
|
|
73
|
+
raise RedirectLimitReached, "Too many redirects (status #{raw_response.status})"
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
retry_from_terminal_redirect!(raw_response, response_guard, deadline:)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def redirect_limit_reached?(response)
|
|
80
|
+
return false unless response.is_a?(HTTPX::Response)
|
|
81
|
+
return false if response.status == 304
|
|
82
|
+
return false unless (300..399).cover?(response.status)
|
|
83
|
+
|
|
84
|
+
location = response.headers['location']
|
|
85
|
+
!location.nil? && !location.strip.empty?
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def terminal_redirect_url(raw_response)
|
|
89
|
+
location = raw_response.headers['location']
|
|
90
|
+
if location && !location.empty?
|
|
91
|
+
base = raw_response.uri || request_url
|
|
92
|
+
return normalize_url(URI.join(base.to_s, location))
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
@last_redirect_to
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def terminal_redirect_retryable?(raw_response)
|
|
99
|
+
return false if @terminal_redirect_retried
|
|
100
|
+
|
|
101
|
+
target = terminal_redirect_url(raw_response)
|
|
102
|
+
target && target.to_s != request_url.to_s
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def retry_from_terminal_redirect!(raw_response, response_guard, deadline:)
|
|
106
|
+
terminal_url = terminal_redirect_url(raw_response)
|
|
107
|
+
@terminal_redirect_retried = true
|
|
108
|
+
Log.debug("#{self.class}: redirect limit reached; retrying once from #{terminal_url}")
|
|
109
|
+
begin_terminal_url_request!(terminal_url)
|
|
110
|
+
new_response = execute_http_request(response_guard, deadline:, consume_budget: false)
|
|
111
|
+
if redirect_limit_reached?(new_response)
|
|
112
|
+
raise RedirectLimitReached, "Too many redirects (status #{new_response.status})"
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
new_response
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def begin_terminal_url_request!(terminal_url)
|
|
119
|
+
ctx.policy.validate_request!(url: terminal_url, origin_url: ctx.origin_url, relation: ctx.relation)
|
|
120
|
+
@request_url_override = terminal_url
|
|
121
|
+
@last_redirect_to = nil
|
|
122
|
+
@current_url = terminal_url
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def request_url
|
|
126
|
+
@request_url_override || ctx.url
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def execute_http_request(response_guard, deadline:, consume_budget: true)
|
|
130
|
+
preflight!(consume_budget:)
|
|
131
|
+
session = build_session(response_guard, deadline:)
|
|
132
|
+
response = session.get(request_url.to_s, headers: ctx.headers)
|
|
133
|
+
raise response.error if response.is_a?(HTTPX::ErrorResponse)
|
|
134
|
+
|
|
135
|
+
response
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def build_session(_response_guard, deadline:)
|
|
139
|
+
session_client(deadline)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def session_client(deadline)
|
|
143
|
+
session = ctx.policy.allow_private_networks? ? self.class.base_session : self.class.base_ssrf_session
|
|
144
|
+
session.with(session_options(deadline))
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def session_options(deadline)
|
|
148
|
+
remaining = remaining_timeout_seconds(deadline)
|
|
149
|
+
{
|
|
150
|
+
timeout: session_timeouts(remaining),
|
|
151
|
+
max_redirects: ctx.policy.max_redirects,
|
|
152
|
+
follow_insecure_redirects: false,
|
|
153
|
+
max_response_body_size: ctx.policy.max_response_bytes,
|
|
154
|
+
resolver_class: :system,
|
|
155
|
+
redirect_on: redirect_callback
|
|
156
|
+
}
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def session_timeouts(remaining)
|
|
160
|
+
{
|
|
161
|
+
connect_timeout: [ctx.policy.connect_timeout_seconds, remaining].min,
|
|
162
|
+
read_timeout: [ctx.policy.read_timeout_seconds, remaining].min,
|
|
163
|
+
operation_timeout: remaining,
|
|
164
|
+
request_timeout: remaining,
|
|
165
|
+
total_request_timeout: remaining
|
|
166
|
+
}
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def redirect_callback
|
|
170
|
+
lambda do |redirect_uri|
|
|
171
|
+
to_url = normalize_url(redirect_uri)
|
|
172
|
+
from_url = @current_url
|
|
173
|
+
@last_redirect_to = to_url
|
|
174
|
+
ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
|
|
175
|
+
@current_url = to_url
|
|
176
|
+
true
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def build_response(response)
|
|
181
|
+
headers = response.headers.to_h
|
|
182
|
+
Response.new(
|
|
183
|
+
body: CompressedBody.decode(response.body.to_s, headers:),
|
|
184
|
+
headers:,
|
|
185
|
+
url: response_url(response),
|
|
186
|
+
status: response.status
|
|
187
|
+
)
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def response_url(response)
|
|
191
|
+
return ctx.url unless (uri = response.uri)
|
|
192
|
+
|
|
193
|
+
Html2rss::Url.from_absolute(uri.to_s)
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def normalize_url(url)
|
|
197
|
+
Html2rss::Url.from_absolute(url.to_s)
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def remaining_timeout_seconds(deadline)
|
|
201
|
+
remaining = deadline - monotonic_now
|
|
202
|
+
raise RequestTimedOut, 'Request timed out' if remaining <= 0
|
|
203
|
+
|
|
204
|
+
remaining
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def monotonic_now
|
|
208
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
def timeout_error?(error)
|
|
212
|
+
error.is_a?(HTTPX::TimeoutError) || super
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def ssrf_error?(error)
|
|
216
|
+
error.is_a?(HTTPX::ServerSideRequestForgeryError) || super
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def response_too_large_error?(error)
|
|
220
|
+
(error.is_a?(HTTPX::Error) && error.message.include?('maximum response body size exceeded')) || super
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def connection_error?(error)
|
|
224
|
+
error.is_a?(HTTPX::ConnectionError) || error.is_a?(HTTPX::TLSError) || super
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
end
|
|
@@ -84,11 +84,10 @@ module Html2rss
|
|
|
84
84
|
# Returns the default request policy.
|
|
85
85
|
#
|
|
86
86
|
# @return [Policy] a default, frozen policy instance
|
|
87
|
-
# rubocop:disable Layout/ClassStructure
|
|
87
|
+
# rubocop:disable-next Layout/ClassStructure
|
|
88
88
|
def self.default
|
|
89
89
|
new
|
|
90
90
|
end
|
|
91
|
-
# rubocop:enable Layout/ClassStructure
|
|
92
91
|
|
|
93
92
|
##
|
|
94
93
|
# Validates whether a request target is permitted for the given context.
|
|
@@ -31,7 +31,7 @@ module Html2rss
|
|
|
31
31
|
# @param status [Integer, nil] the HTTP status code when available
|
|
32
32
|
# @param transport_meta [Hash] allowlisted upstream telemetry (frozen when present)
|
|
33
33
|
# @param captured_responses [Array<Hash>] JSON XHR/fetch bodies captured during browser scrapes
|
|
34
|
-
# rubocop:disable Metrics/MethodLength, Metrics/ParameterLists -- transport + capture fields stay co-located
|
|
34
|
+
# rubocop:disable-next Metrics/MethodLength, Metrics/ParameterLists -- transport + capture fields stay co-located
|
|
35
35
|
def initialize(body:, url:, headers: {}, status: nil, transport_meta: EMPTY_TRANSPORT_META,
|
|
36
36
|
captured_responses: EMPTY_CAPTURED_RESPONSES)
|
|
37
37
|
@body = body
|
|
@@ -50,7 +50,6 @@ module Html2rss
|
|
|
50
50
|
captured_responses.freeze
|
|
51
51
|
end
|
|
52
52
|
end
|
|
53
|
-
# rubocop:enable Metrics/MethodLength, Metrics/ParameterLists
|
|
54
53
|
|
|
55
54
|
# @return [String] the raw body of the response
|
|
56
55
|
attr_reader :body
|
|
@@ -160,12 +159,12 @@ module Html2rss
|
|
|
160
159
|
return false if ct.empty? || ct.include?('html') || ct.include?('json')
|
|
161
160
|
|
|
162
161
|
# Substring match against Content-Type (not Array#intersect? on a String).
|
|
163
|
-
FEED_CT_MARKERS.any? { |marker| ct.include?(marker) }
|
|
162
|
+
FEED_CT_MARKERS.any? { |marker| ct.include?(marker) }
|
|
164
163
|
end
|
|
165
164
|
|
|
166
165
|
def feed_looking_body?
|
|
167
166
|
snippet = body.to_s[0, 800].downcase
|
|
168
|
-
FEED_BODY_MARKERS.any? { |marker| snippet.include?(marker) }
|
|
167
|
+
FEED_BODY_MARKERS.any? { |marker| snippet.include?(marker) }
|
|
169
168
|
end
|
|
170
169
|
end
|
|
171
170
|
end
|
|
@@ -9,22 +9,6 @@ module Html2rss
|
|
|
9
9
|
# @param policy [Policy] request policy that defines byte ceilings
|
|
10
10
|
def initialize(policy:)
|
|
11
11
|
@policy = policy
|
|
12
|
-
@streamed_bytes = 0
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
##
|
|
16
|
-
# Validates response headers and streamed byte count.
|
|
17
|
-
#
|
|
18
|
-
# @param total_bytes [Integer] cumulative byte count received so far
|
|
19
|
-
# @param headers [Hash, nil] response headers if known
|
|
20
|
-
# @return [void]
|
|
21
|
-
# @raise [ResponseTooLarge] if the response exceeds configured limits
|
|
22
|
-
def inspect_chunk!(total_bytes:, headers: nil)
|
|
23
|
-
header_length = headers&.fetch('content-length', headers&.fetch('Content-Length', nil))
|
|
24
|
-
raise_if_too_large!(header_length.to_i, policy.max_response_bytes) if header_length
|
|
25
|
-
|
|
26
|
-
@streamed_bytes = total_bytes
|
|
27
|
-
raise_if_too_large!(@streamed_bytes, policy.max_response_bytes)
|
|
28
12
|
end
|
|
29
13
|
|
|
30
14
|
##
|
|
@@ -97,11 +97,16 @@ module Html2rss
|
|
|
97
97
|
# @param error [StandardError]
|
|
98
98
|
# @return [void]
|
|
99
99
|
# @raise [StandardError]
|
|
100
|
+
# rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- error translation dispatch
|
|
100
101
|
def handle_error(error)
|
|
101
102
|
if timeout_error?(error)
|
|
102
103
|
log_timeout!(reason: 'transport')
|
|
103
104
|
Log.debug("#{self.class}: transport timeout message=#{error.message}")
|
|
104
105
|
raise RequestTimedOut, error.message
|
|
106
|
+
elsif ssrf_error?(error)
|
|
107
|
+
raise PrivateNetworkDenied, error.message
|
|
108
|
+
elsif response_too_large_error?(error)
|
|
109
|
+
raise ResponseTooLarge, "Response exceeded #{ctx.policy.max_response_bytes} bytes"
|
|
105
110
|
elsif connection_error?(error)
|
|
106
111
|
translate_connection_error(error)
|
|
107
112
|
else
|
|
@@ -111,7 +116,7 @@ module Html2rss
|
|
|
111
116
|
|
|
112
117
|
# @param reason [String] timeout classification (budget_exhausted / transport)
|
|
113
118
|
# @return [void]
|
|
114
|
-
# rubocop:disable Metrics/AbcSize -- structured timeout fields stay in one log line
|
|
119
|
+
# rubocop:disable-next Metrics/AbcSize -- structured timeout fields stay in one log line
|
|
115
120
|
def log_timeout!(reason:)
|
|
116
121
|
remaining = ctx.budget.remaining_timeout_seconds
|
|
117
122
|
remaining_label = remaining.nil? ? 'untracked' : format('%.3f', remaining)
|
|
@@ -124,7 +129,6 @@ module Html2rss
|
|
|
124
129
|
]
|
|
125
130
|
Log.info("#{self.class}: request timeout #{detail.join(' ')}")
|
|
126
131
|
end
|
|
127
|
-
# rubocop:enable Metrics/AbcSize
|
|
128
132
|
|
|
129
133
|
# @param error [StandardError]
|
|
130
134
|
# @return [void]
|
|
@@ -136,14 +140,25 @@ module Html2rss
|
|
|
136
140
|
# @param error [StandardError]
|
|
137
141
|
# @return [Boolean]
|
|
138
142
|
def timeout_error?(error)
|
|
139
|
-
error.is_a?(
|
|
140
|
-
|
|
143
|
+
error.is_a?(Timeout::Error)
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# @param _error [StandardError]
|
|
147
|
+
# @return [Boolean]
|
|
148
|
+
def ssrf_error?(_error)
|
|
149
|
+
false
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# @param _error [StandardError]
|
|
153
|
+
# @return [Boolean]
|
|
154
|
+
def response_too_large_error?(_error)
|
|
155
|
+
false
|
|
141
156
|
end
|
|
142
157
|
|
|
143
158
|
# @param error [StandardError]
|
|
144
159
|
# @return [Boolean]
|
|
145
160
|
def connection_error?(error)
|
|
146
|
-
error.is_a?(
|
|
161
|
+
error.is_a?(SocketError) || error.is_a?(SystemCallError)
|
|
147
162
|
end
|
|
148
163
|
end
|
|
149
164
|
end
|
|
@@ -1,18 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'singleton'
|
|
4
|
-
require 'forwardable'
|
|
5
|
-
|
|
6
3
|
module Html2rss
|
|
7
4
|
##
|
|
8
5
|
# Requests website URLs to retrieve their HTML for further processing.
|
|
9
|
-
# Provides concrete transport strategies (e.g.
|
|
6
|
+
# Provides concrete transport strategies (e.g. HttpxStrategy, BotasaurusStrategy).
|
|
10
7
|
#
|
|
11
8
|
# Feed-level +:auto+ is not registered here — {FeedPipeline::StrategyPlan} resolves
|
|
12
9
|
# it to a concrete strategy (or {FeedPipeline::AutoFallback} chain) before execute.
|
|
13
10
|
class RequestService
|
|
14
|
-
include Singleton
|
|
15
|
-
|
|
16
11
|
# Raised when an unknown request strategy is requested.
|
|
17
12
|
class UnknownStrategy < Html2rss::Error; end
|
|
18
13
|
# Raised when a URL cannot be parsed or validated.
|
|
@@ -29,6 +24,8 @@ module Html2rss
|
|
|
29
24
|
class CrossOriginFollowUpDenied < Html2rss::Error; end
|
|
30
25
|
# Raised when a response exceeds configured size limits.
|
|
31
26
|
class ResponseTooLarge < Html2rss::Error; end
|
|
27
|
+
# Raised when HTTP redirect limits are exceeded.
|
|
28
|
+
class RedirectLimitReached < Html2rss::Error; end
|
|
32
29
|
# Raised when blocked content surfaces are detected.
|
|
33
30
|
class BlockedSurfaceDetected < Html2rss::Error; end
|
|
34
31
|
|
|
@@ -57,94 +54,60 @@ module Html2rss
|
|
|
57
54
|
# Raised when Botasaurus responds but the scrape fails (upstream error, bad payload).
|
|
58
55
|
class BotasaurusServiceError < Html2rss::Error; end
|
|
59
56
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
execute].each do |method|
|
|
70
|
-
def_delegator :instance, method
|
|
71
|
-
end
|
|
72
|
-
end
|
|
73
|
-
|
|
74
|
-
def initialize
|
|
75
|
-
@strategies = {
|
|
76
|
-
faraday: FaradayStrategy,
|
|
77
|
-
botasaurus: BotasaurusStrategy,
|
|
78
|
-
local_file: LocalFileStrategy
|
|
79
|
-
}
|
|
80
|
-
@default_strategy_name = :faraday
|
|
81
|
-
end
|
|
57
|
+
# Map of supported strategy names to their implementation classes.
|
|
58
|
+
# @return [Hash{Symbol => Class<Strategy>}]
|
|
59
|
+
STRATEGIES = {
|
|
60
|
+
default: HttpxStrategy,
|
|
61
|
+
httpx: HttpxStrategy,
|
|
62
|
+
faraday: HttpxStrategy,
|
|
63
|
+
botasaurus: BotasaurusStrategy,
|
|
64
|
+
local_file: LocalFileStrategy
|
|
65
|
+
}.freeze
|
|
82
66
|
|
|
83
|
-
#
|
|
84
|
-
|
|
67
|
+
# Canonical default strategy symbol.
|
|
68
|
+
# @return [Symbol]
|
|
69
|
+
DEFAULT_STRATEGY_NAME = :default
|
|
85
70
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
# @return [Symbol] the selected default strategy name
|
|
90
|
-
# @raise [UnknownStrategy] if the strategy is not registered
|
|
91
|
-
def default_strategy_name=(strategy)
|
|
92
|
-
raise UnknownStrategy unless strategy_registered?(strategy)
|
|
93
|
-
|
|
94
|
-
@default_strategy_name = strategy.to_sym
|
|
95
|
-
end
|
|
71
|
+
class << self
|
|
72
|
+
# @return [Symbol] the default strategy name
|
|
73
|
+
def default_strategy_name = DEFAULT_STRATEGY_NAME
|
|
96
74
|
|
|
97
|
-
|
|
98
|
-
|
|
75
|
+
# @return [Array<String>] the names of the registered strategies
|
|
76
|
+
def strategy_names = STRATEGIES.keys.map(&:to_s)
|
|
99
77
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
def register_strategy(name, strategy_class)
|
|
107
|
-
unless strategy_class.is_a?(Class)
|
|
108
|
-
raise ArgumentError, "Expected a Class for strategy, got #{strategy_class.class}"
|
|
78
|
+
##
|
|
79
|
+
# Checks if a strategy is registered.
|
|
80
|
+
# @param name [Symbol, String] the name of the strategy
|
|
81
|
+
# @return [Boolean] true if the strategy is registered, false otherwise.
|
|
82
|
+
def strategy_registered?(name)
|
|
83
|
+
STRATEGIES.key?(name.to_sym)
|
|
109
84
|
end
|
|
110
85
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
def unregister_strategy(name) # rubocop:disable Naming/PredicateMethod
|
|
128
|
-
name_sym = name.to_sym
|
|
129
|
-
raise ArgumentError, 'Cannot unregister the default strategy.' if name_sym == @default_strategy_name
|
|
86
|
+
##
|
|
87
|
+
# Executes the request using the specified strategy.
|
|
88
|
+
# @param ctx [Context] the context for the request.
|
|
89
|
+
# @param strategy [Symbol, String] the strategy to use (defaults to the default strategy).
|
|
90
|
+
# @return [Response] the response from the executed strategy.
|
|
91
|
+
# @raise [UnknownStrategy] if the strategy is not registered.
|
|
92
|
+
def execute(ctx, strategy: default_strategy_name)
|
|
93
|
+
strategy_sym = strategy.to_sym
|
|
94
|
+
strategy_class = STRATEGIES.fetch(strategy_sym) do
|
|
95
|
+
raise UnknownStrategy,
|
|
96
|
+
"The strategy '#{strategy}' is not known. Available strategies: #{strategy_names.join(', ')}"
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
warn_migration_strategy(strategy_sym) if strategy_sym == :faraday
|
|
100
|
+
strategy_class.new(ctx).execute
|
|
101
|
+
end
|
|
130
102
|
|
|
131
|
-
|
|
132
|
-
end
|
|
103
|
+
private
|
|
133
104
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
# @raise [ArgumentError] if the context is nil.
|
|
140
|
-
# @raise [UnknownStrategy] if the strategy is not registered.
|
|
141
|
-
def execute(ctx, strategy: default_strategy_name)
|
|
142
|
-
strategy_class = @strategies.fetch(strategy.to_sym) do
|
|
143
|
-
raise UnknownStrategy,
|
|
144
|
-
"The strategy '#{strategy}' is not known. Available strategies: #{strategy_names.join(', ')}"
|
|
105
|
+
def warn_migration_strategy(strategy)
|
|
106
|
+
message = "RequestService: strategy ':#{strategy}' is deprecated for migration and will be removed " \
|
|
107
|
+
"in a future release. Use ':default' instead."
|
|
108
|
+
warn(message, category: :deprecated)
|
|
109
|
+
Log.warn(message)
|
|
145
110
|
end
|
|
146
|
-
|
|
147
|
-
strategy_class.new(ctx).execute
|
|
148
111
|
end
|
|
149
112
|
end
|
|
150
113
|
end
|