simple-rss 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +174 -0
- data/README.md +492 -8
- data/Rakefile +1 -1
- data/examples/digest.rb +19 -0
- data/examples/discover.rb +14 -0
- data/examples/feedbag.rb +10 -0
- data/lib/simple-rss/discovery.rb +107 -0
- data/lib/simple-rss/entry_normalizer.rb +356 -0
- data/lib/simple-rss/http_client.rb +216 -0
- data/lib/simple-rss/json_entry_normalizer.rb +147 -0
- data/lib/simple-rss/json_feed.rb +176 -0
- data/lib/simple-rss/normalized_entry.rb +79 -0
- data/lib/simple-rss/request_errors.rb +21 -0
- data/lib/simple-rss/request_policy.rb +72 -0
- data/lib/simple-rss/xml_element.rb +136 -0
- data/lib/simple-rss.rb +225 -77
- data/simple-rss.gemspec +5 -5
- data/test/base/category_parsing_test.rb +217 -0
- data/test/base/date_ordering_test.rb +95 -0
- data/test/base/discovery_dependency_test.rb +29 -0
- data/test/base/discovery_test.rb +151 -0
- data/test/base/discovery_transport_test.rb +413 -0
- data/test/base/enumerable_test.rb +16 -0
- data/test/base/feedbag_integration_test.rb +21 -0
- data/test/base/json_feed_test.rb +372 -0
- data/test/base/normalized_entries_test.rb +410 -0
- data/test/base/normalized_fetch_test.rb +132 -0
- data/test/base/relation_links_test.rb +162 -0
- data/test/data/atom_categories.xml +23 -0
- data/test/data/atom_nested_link.xml +13 -0
- data/test/data/discovery.html +28 -0
- data/test/data/json_feed_1.json +75 -0
- data/test/data/json_feed_1_1.json +78 -0
- data/test/data/mixed_dates.xml +55 -0
- data/test/data/normalized_atom.xml +22 -0
- data/test/data/normalized_rss.xml +21 -0
- data/test/data/rss_categories.xml +15 -0
- data/test/support/http_server.rb +44 -0
- data/test/support/replace_method.rb +14 -0
- metadata +42 -10
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
require "test_helper"
|
|
2
|
+
require "simple-rss/http_client"
|
|
3
|
+
require "zlib"
|
|
4
|
+
require "stringio"
|
|
5
|
+
require_relative "../support/http_server"
|
|
6
|
+
require_relative "../support/replace_method"
|
|
7
|
+
|
|
8
|
+
class DiscoveryTransportTest < Test::Unit::TestCase
|
|
9
|
+
include HTTPServer
|
|
10
|
+
include ReplaceMethod
|
|
11
|
+
|
|
12
|
+
PAGE = [200, { "Content-Type" => "text/html" }, "<html><head></head><body>Example</body></html>"].freeze
|
|
13
|
+
FEED = '<rss version="2.0"><channel><title>Example</title></channel></rss>'.freeze
|
|
14
|
+
|
|
15
|
+
def test_public_policy_rejects_prohibited_ipv4_and_ipv6_before_connecting
|
|
16
|
+
prohibited = %w[
|
|
17
|
+
0.0.0.0 10.0.0.1 100.64.0.1 127.0.0.1 169.254.169.254 172.16.0.1 192.0.0.1 192.0.2.1
|
|
18
|
+
192.88.99.1 192.168.1.1 198.18.0.1 198.51.100.1 203.0.113.1 224.0.0.1 255.255.255.255
|
|
19
|
+
[::] [::1] [::ffff:127.0.0.1] [64:ff9b::7f00:1] [100::1] [2001::1] [2001:db8::1]
|
|
20
|
+
[2002:7f00:1::] [3fff::1] [fc00::1] [fe80::1] [ff02::1]
|
|
21
|
+
]
|
|
22
|
+
with_replaced_method(TCPSocket, :open, ->(*) { flunk "Prohibited address reached the socket" }) do
|
|
23
|
+
prohibited.each do |host|
|
|
24
|
+
assert_raise(SimpleRSS::PolicyError, host) { SimpleRSS.discover("http://#{host}/", timeout: 1) }
|
|
25
|
+
assert_raise(SimpleRSS::PolicyError, host) { SimpleRSS.discover(host, timeout: 1) }
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def test_malformed_urls_and_schemes_fail_before_connecting
|
|
31
|
+
with_replaced_method(TCPSocket, :open, ->(*) { flunk "Invalid URL reached the socket" }) do
|
|
32
|
+
[nil, 123, [], {}, "", "/relative", "ftp://example.com/feed", "file:///tmp/feed", "mailto:reader@example.com", "javascript:alert(1)", "http://", "http://[broken", "http://example.com:0", "http://example.com:65536", "https://user:password@example.com/", "user:password@example.com"].each do |url|
|
|
33
|
+
assert_raise(SimpleRSS::PolicyError, url.inspect) { SimpleRSS.discover(url) }
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def test_mixed_public_and_private_dns_answers_are_rejected
|
|
39
|
+
with_replaced_method(Resolv, :getaddresses, ->(_host) { ["8.8.8.8", "127.0.0.1"] }) do
|
|
40
|
+
with_replaced_method(TCPSocket, :open, ->(*) { flunk "Mixed DNS answers reached the socket" }) do
|
|
41
|
+
assert_raise(SimpleRSS::PolicyError) { SimpleRSS.discover("http://site.example/") }
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def test_checked_dns_address_is_pinned_and_host_header_is_preserved
|
|
47
|
+
resolutions = 0
|
|
48
|
+
resolver = lambda do |host|
|
|
49
|
+
assert_equal "site.example", host
|
|
50
|
+
resolutions += 1
|
|
51
|
+
resolutions == 1 ? ["8.8.8.8"] : ["127.0.0.1"]
|
|
52
|
+
end
|
|
53
|
+
with_server([PAGE]) do |local_url, requests|
|
|
54
|
+
with_pinned_connection(local_url, resolver) do |connections|
|
|
55
|
+
assert_empty SimpleRSS.discover("http://site.example/", timeout: 1)
|
|
56
|
+
assert_equal [["8.8.8.8", 80]], connections
|
|
57
|
+
assert_equal 1, resolutions
|
|
58
|
+
assert_include requests.first, "Host: site.example"
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def test_dns_policy_is_rechecked_on_each_redirect
|
|
64
|
+
resolutions = 0
|
|
65
|
+
resolver = lambda do |_host|
|
|
66
|
+
resolutions += 1
|
|
67
|
+
resolutions == 1 ? ["8.8.8.8"] : ["127.0.0.1"]
|
|
68
|
+
end
|
|
69
|
+
with_server([[302, { "Location" => "/next" }, ""]]) do |local_url, requests|
|
|
70
|
+
with_pinned_connection(local_url, resolver) do |connections|
|
|
71
|
+
assert_raise(SimpleRSS::PolicyError) { SimpleRSS.discover("http://site.example/", timeout: 1) }
|
|
72
|
+
assert_equal 2, resolutions
|
|
73
|
+
assert_equal 1, connections.size
|
|
74
|
+
assert_equal 1, requests.size
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def test_redirect_to_a_private_literal_is_blocked
|
|
80
|
+
with_server([[302, { "Location" => "http://127.0.0.1:9/feed" }, ""]]) do |local_url, requests|
|
|
81
|
+
with_pinned_connection(local_url, ->(_host) { ["8.8.8.8"] }) do |connections|
|
|
82
|
+
assert_raise(SimpleRSS::PolicyError) { SimpleRSS.discover("http://site.example/", timeout: 1) }
|
|
83
|
+
assert_equal 1, connections.size
|
|
84
|
+
assert_equal 1, requests.size
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def test_public_ipv6_addresses_are_checked_and_pinned
|
|
90
|
+
with_server([PAGE]) do |local_url, _requests|
|
|
91
|
+
with_pinned_connection(local_url, ->(_host) { ["2606:4700:4700::1111"] }) do |connections|
|
|
92
|
+
assert_empty SimpleRSS.discover("http://site.example/", timeout: 1)
|
|
93
|
+
assert_equal [["2606:4700:4700::1111", 80]], connections
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def test_an_application_policy_can_allow_one_internal_destination
|
|
99
|
+
with_server([PAGE]) do |url, requests|
|
|
100
|
+
policy = ->(uri, address) { uri.hostname == "127.0.0.1" && IPAddr.new("127.0.0.1/32").include?(address) }
|
|
101
|
+
assert_empty SimpleRSS.discover(url, network_policy: policy, timeout: 1)
|
|
102
|
+
assert_equal 1, requests.size
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def test_cross_origin_redirects_strip_credentials_and_keep_safe_headers
|
|
107
|
+
headers = { "Authorization" => "Bearer fixture", "Cookie" => "session=fixture", "X-Api-Key" => "fixture", "User-Agent" => "Feed tests", "Accept" => "text/html", "Accept-Language" => "en" }
|
|
108
|
+
options = { network_policy: :unrestricted, timeout: 1, headers: headers, etag: '"fixture"', last_modified: "Sat, 12 Sep 2026 10:00:00 GMT" }
|
|
109
|
+
with_server([PAGE]) do |destination, destination_requests|
|
|
110
|
+
with_server([[302, { "Location" => destination }, ""]]) do |source, source_requests|
|
|
111
|
+
assert_empty SimpleRSS.discover(source, options)
|
|
112
|
+
assert_include source_requests.first, "Authorization: Bearer fixture"
|
|
113
|
+
assert_include source_requests.first, "Cookie: session=fixture"
|
|
114
|
+
assert_include source_requests.first, 'If-None-Match: "fixture"'
|
|
115
|
+
assert_empty destination_requests.first.grep(/Authorization:|Cookie:|Api-Key:|If-None-Match:|If-Modified-Since:/i)
|
|
116
|
+
assert_include destination_requests.first, "User-Agent: Feed tests"
|
|
117
|
+
assert_include destination_requests.first, "Accept: text/html"
|
|
118
|
+
assert_include destination_requests.first, "Accept-Language: en"
|
|
119
|
+
assert_equal "Bearer fixture", options[:headers]["Authorization"]
|
|
120
|
+
assert_equal '"fixture"', options[:etag]
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def test_same_origin_redirects_keep_authorization
|
|
126
|
+
with_server([[302, { "Location" => "/next" }, ""], PAGE]) do |url, requests|
|
|
127
|
+
SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1, headers: { "Authorization" => "Bearer fixture" })
|
|
128
|
+
requests.each { |request| assert_include request, "Authorization: Bearer fixture" }
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def test_redirect_loops_and_limits_are_bounded
|
|
133
|
+
with_server([[302, { "Location" => "/" }, ""]]) do |url, requests|
|
|
134
|
+
assert_raise(SimpleRSS::RedirectError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1) }
|
|
135
|
+
assert_equal 1, requests.size
|
|
136
|
+
end
|
|
137
|
+
with_server([[302, { "Location" => "/next" }, ""]]) do |url, requests|
|
|
138
|
+
assert_raise(SimpleRSS::RedirectError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1, max_redirects: 0) }
|
|
139
|
+
assert_equal 1, requests.size
|
|
140
|
+
end
|
|
141
|
+
["file:///tmp/feed", "ftp://example.com/feed", "http://[invalid", "https://user:password@example.com/"].each do |location|
|
|
142
|
+
with_server([[302, { "Location" => location }, ""]]) do |url, requests|
|
|
143
|
+
assert_raise(SimpleRSS::PolicyError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1) }
|
|
144
|
+
assert_equal 1, requests.size
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def test_bodyless_conditional_responses_ignore_representation_encoding
|
|
150
|
+
response = [304, { "Content-Encoding" => "gzip", "ETag" => '"fixture"' }, ""]
|
|
151
|
+
with_server([response, response]) do |url, _requests|
|
|
152
|
+
assert_nil SimpleRSS.fetch(url, network_policy: :unrestricted, timeout: 1, max_bytes: 1, etag: '"fixture"')
|
|
153
|
+
error = assert_raise(SimpleRSS::HTTPError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1) }
|
|
154
|
+
assert_equal 304, error.status_code
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def test_truncated_content_length_is_a_transport_failure
|
|
159
|
+
response = lambda do |client, _request|
|
|
160
|
+
client.write("HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: #{PAGE.last.bytesize + 20}\r\nConnection: close\r\n\r\n#{PAGE.last}")
|
|
161
|
+
end
|
|
162
|
+
with_server([response]) do |url, _requests|
|
|
163
|
+
assert_raise(SimpleRSS::RequestError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1) }
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def test_no_follow_option_reports_the_redirect_status
|
|
168
|
+
with_server([[302, { "Location" => "/next" }, ""]]) do |url, requests|
|
|
169
|
+
error = assert_raise(SimpleRSS::HTTPError) { SimpleRSS.discover(url, network_policy: :unrestricted, follow_redirects: false, timeout: 1) }
|
|
170
|
+
assert_equal 302, error.status_code
|
|
171
|
+
assert_equal 1, requests.size
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def test_response_byte_limit_accepts_the_boundary_and_rejects_overflow
|
|
176
|
+
with_server([PAGE, PAGE]) do |url, requests|
|
|
177
|
+
assert_empty SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1, max_bytes: PAGE.last.bytesize)
|
|
178
|
+
assert_raise(SimpleRSS::ResponseTooLarge) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1, max_bytes: PAGE.last.bytesize - 1) }
|
|
179
|
+
assert_equal 2, requests.size
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def test_chunked_body_is_stopped_before_its_end
|
|
184
|
+
response = lambda do |client, _request|
|
|
185
|
+
client.write("HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nTransfer-Encoding: chunked\r\n\r\n100\r\n" + ("x" * 256) + "\r\n")
|
|
186
|
+
wait_for_disconnect(client)
|
|
187
|
+
end
|
|
188
|
+
with_server([response]) do |url, requests|
|
|
189
|
+
assert_raise(SimpleRSS::ResponseTooLarge) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1, max_bytes: 128) }
|
|
190
|
+
assert_equal 1, requests.size
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def test_gzip_and_deflate_are_decoded_with_a_decompressed_size_limit
|
|
195
|
+
{ "gzip" => gzip(PAGE.last), "deflate" => Zlib::Deflate.deflate(PAGE.last) }.each do |encoding, compressed|
|
|
196
|
+
with_server([[200, PAGE[1].merge("Content-Encoding" => encoding), compressed]]) do |url, _requests|
|
|
197
|
+
assert_empty SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1)
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
compressed = gzip("<html><body>" + ("x" * 200_000) + "</body></html>")
|
|
201
|
+
assert_operator compressed.bytesize, :<, 1024
|
|
202
|
+
with_server([[200, PAGE[1].merge("Content-Encoding" => "gzip"), compressed]]) do |url, _requests|
|
|
203
|
+
assert_raise(SimpleRSS::ResponseTooLarge) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1, max_bytes: 1024) }
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def test_body_budget_is_shared_across_redirects
|
|
208
|
+
responses = [[302, { "Location" => "/next" }, "x" * 100], PAGE]
|
|
209
|
+
with_server(responses) do |url, requests|
|
|
210
|
+
assert_raise(SimpleRSS::ResponseTooLarge) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1, max_bytes: 100 + PAGE.last.bytesize - 1) }
|
|
211
|
+
assert_equal 2, requests.size
|
|
212
|
+
end
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def test_invalid_compression_and_partial_responses_fail_clearly
|
|
216
|
+
responses = [
|
|
217
|
+
[200, PAGE[1].merge("Content-Encoding" => "br"), "wrong"],
|
|
218
|
+
[200, PAGE[1].merge("Content-Encoding" => "gzip"), "wrong"],
|
|
219
|
+
[200, PAGE[1].merge("Content-Encoding" => "gzip"), gzip(PAGE.last).byteslice(0, 15)],
|
|
220
|
+
[200, PAGE[1].merge("Content-Encoding" => "gzip"), gzip(PAGE.last) + gzip("ignored")],
|
|
221
|
+
[206, PAGE[1].merge("Content-Range" => "bytes 0-9/100"), "0123456789"]
|
|
222
|
+
]
|
|
223
|
+
responses.each do |response|
|
|
224
|
+
with_server([response]) do |url, _requests|
|
|
225
|
+
assert_raise(SimpleRSS::RequestError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1) }
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
def test_read_timeout_is_visible_and_does_not_retry
|
|
231
|
+
with_server([->(client, _request) { wait_for_disconnect(client) }]) do |url, requests|
|
|
232
|
+
assert_raise(SimpleRSS::RequestTimeout) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 0.1) }
|
|
233
|
+
assert_equal 1, requests.size
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
def test_dns_resolution_is_inside_the_total_timeout
|
|
238
|
+
with_replaced_method(Resolv, :getaddresses, ->(_host) { Queue.new.pop }) do
|
|
239
|
+
assert_raise(SimpleRSS::RequestTimeout) { SimpleRSS.discover("http://site.example/", timeout: 0.1) }
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
def test_missing_dns_results_and_connection_failures_are_not_empty_candidates
|
|
244
|
+
with_replaced_method(Resolv, :getaddresses, ->(_host) { [] }) do
|
|
245
|
+
assert_raise(SimpleRSS::RequestError) { SimpleRSS.discover("http://site.example/") }
|
|
246
|
+
end
|
|
247
|
+
with_replaced_method(Resolv, :getaddresses, ->(_host) { ["8.8.8.8"] }) do
|
|
248
|
+
with_replaced_method(TCPSocket, :open, ->(*) { raise Errno::ECONNREFUSED }) do
|
|
249
|
+
assert_raise(SimpleRSS::RequestError) { SimpleRSS.discover("http://site.example/") }
|
|
250
|
+
end
|
|
251
|
+
end
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
def test_policy_disables_environment_proxies_and_keeps_tls_verification
|
|
255
|
+
constructor = Net::HTTP.method(:new)
|
|
256
|
+
sessions = []
|
|
257
|
+
replacement = lambda do |*arguments|
|
|
258
|
+
assert_equal ["site.example", 443, nil], arguments
|
|
259
|
+
session = constructor.call(*arguments)
|
|
260
|
+
sessions << session
|
|
261
|
+
session
|
|
262
|
+
end
|
|
263
|
+
with_replaced_method(Resolv, :getaddresses, ->(_host) { ["8.8.8.8"] }) do
|
|
264
|
+
with_replaced_method(Net::HTTP, :new, replacement) do
|
|
265
|
+
with_replaced_method(TCPSocket, :open, ->(*) { raise Errno::ECONNREFUSED }) do
|
|
266
|
+
assert_raise(SimpleRSS::RequestError) { SimpleRSS.discover("https://site.example/", timeout: 1) }
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
assert_equal false, sessions.first.proxy?
|
|
271
|
+
assert_equal "site.example", sessions.first.address
|
|
272
|
+
assert_equal "8.8.8.8", sessions.first.ipaddr
|
|
273
|
+
assert_equal OpenSSL::SSL::VERIFY_PEER, sessions.first.verify_mode
|
|
274
|
+
assert_equal true, sessions.first.verify_hostname
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
def test_pinned_https_preserves_certificate_hostname_verification
|
|
278
|
+
certificate, private_key = test_certificate
|
|
279
|
+
store = OpenSSL::X509::Store.new
|
|
280
|
+
store.add_cert(certificate)
|
|
281
|
+
context = OpenSSL::SSL::SSLContext.new
|
|
282
|
+
context.cert = certificate
|
|
283
|
+
context.key = private_key
|
|
284
|
+
server = TCPServer.new("127.0.0.1", 0)
|
|
285
|
+
ssl_server = OpenSSL::SSL::SSLServer.new(server, context)
|
|
286
|
+
requests = []
|
|
287
|
+
worker = Thread.new do
|
|
288
|
+
5.times do
|
|
289
|
+
client = nil
|
|
290
|
+
begin
|
|
291
|
+
client = ssl_server.accept
|
|
292
|
+
request = []
|
|
293
|
+
while (line = client.gets)
|
|
294
|
+
break if line == "\r\n"
|
|
295
|
+
|
|
296
|
+
request << line.strip
|
|
297
|
+
end
|
|
298
|
+
requests << request
|
|
299
|
+
client.write("HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: #{PAGE.last.bytesize}\r\nConnection: close\r\n\r\n#{PAGE.last}")
|
|
300
|
+
rescue OpenSSL::SSL::SSLError
|
|
301
|
+
next
|
|
302
|
+
ensure
|
|
303
|
+
client&.close
|
|
304
|
+
end
|
|
305
|
+
end
|
|
306
|
+
end
|
|
307
|
+
constructor = Net::HTTP.method(:new)
|
|
308
|
+
replacement = lambda do |*arguments|
|
|
309
|
+
session = constructor.call(*arguments)
|
|
310
|
+
session.cert_store = store
|
|
311
|
+
session
|
|
312
|
+
end
|
|
313
|
+
with_replaced_method(Net::HTTP, :new, replacement) do
|
|
314
|
+
with_pinned_connection("http://127.0.0.1:#{server.addr[1]}", ->(_host) { ["8.8.8.8"] }) do |connections|
|
|
315
|
+
assert_empty SimpleRSS.discover("https://site.example/", timeout: 1)
|
|
316
|
+
assert_empty SimpleRSS.discover("site.example", timeout: 1)
|
|
317
|
+
assert_empty SimpleRSS.discover("site.example/blog?edition=1#entries", timeout: 1)
|
|
318
|
+
assert_empty SimpleRSS.discover("//site.example/blog", timeout: 1)
|
|
319
|
+
error = assert_raise(SimpleRSS::RequestError) { SimpleRSS.discover("https://wrong.example/", timeout: 1) }
|
|
320
|
+
assert_kind_of OpenSSL::SSL::SSLError, error.cause
|
|
321
|
+
assert_equal Array.new(5) { ["8.8.8.8", 443] }, connections
|
|
322
|
+
end
|
|
323
|
+
end
|
|
324
|
+
worker.value
|
|
325
|
+
assert_equal ["GET / HTTP/1.1", "GET / HTTP/1.1", "GET /blog?edition=1 HTTP/1.1", "GET /blog HTTP/1.1"], requests.map(&:first)
|
|
326
|
+
assert_include requests.first, "Host: site.example"
|
|
327
|
+
ensure
|
|
328
|
+
worker&.kill
|
|
329
|
+
worker&.join
|
|
330
|
+
server&.close
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
def test_bounded_fetch_reuses_the_policy_and_conditional_get
|
|
334
|
+
headers = { "ETag" => '"fixture"', "Last-Modified" => "Sat, 12 Sep 2026 10:00:00 GMT" }
|
|
335
|
+
with_server([[200, headers, FEED], [304, headers, ""]]) do |url, requests|
|
|
336
|
+
feed = SimpleRSS.fetch(url, network_policy: :unrestricted, timeout: 1)
|
|
337
|
+
assert_equal "Example", feed.title
|
|
338
|
+
assert_equal "#{url}/", feed.source_url
|
|
339
|
+
assert_nil SimpleRSS.fetch(url, network_policy: :unrestricted, timeout: 1, etag: feed.etag, last_modified: feed.last_modified)
|
|
340
|
+
assert_include requests.last, 'If-None-Match: "fixture"'
|
|
341
|
+
assert_include requests.last, "If-Modified-Since: #{headers["Last-Modified"]}"
|
|
342
|
+
end
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
def test_bodyless_success_responses_raise_feed_errors
|
|
346
|
+
with_server([[204, {}, ""]]) do |url, _requests|
|
|
347
|
+
assert_raise(SimpleRSS::DiscoveryError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1) }
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
[{}, { network_policy: :unrestricted }].each do |options|
|
|
351
|
+
with_server([[204, {}, ""]]) do |url, _requests|
|
|
352
|
+
error = assert_raise(SimpleRSSError) { SimpleRSS.fetch(url, options.merge(timeout: 1)) }
|
|
353
|
+
assert_equal "Poorly formatted feed", error.message
|
|
354
|
+
end
|
|
355
|
+
end
|
|
356
|
+
end
|
|
357
|
+
|
|
358
|
+
def test_invalid_options_and_controlled_headers_fail_before_connecting
|
|
359
|
+
options = [
|
|
360
|
+
{ network_policy: nil }, { network_policy: :unknown }, { timeout: 0 }, { timeout: nil }, { timeout: Float::INFINITY },
|
|
361
|
+
{ max_bytes: 0 }, { max_bytes: 1.5 }, { max_redirects: -1 }, { max_redirects: 1.5 }, { headers: [] },
|
|
362
|
+
{ headers: { "X-Value" => "bad\r\nInjected: header" } }, { headers: { "Bad Header" => "value" } }
|
|
363
|
+
]
|
|
364
|
+
options += %w[Host Proxy-Authorization Accept-Encoding Range Connection Transfer-Encoding].map { |name| { headers: { name => "value" } } }
|
|
365
|
+
with_replaced_method(TCPSocket, :open, ->(*) { flunk "Invalid options reached the socket" }) do
|
|
366
|
+
options.each do |override|
|
|
367
|
+
assert_raise(ArgumentError, override.inspect) { SimpleRSS.discover("https://site.example/", override) }
|
|
368
|
+
end
|
|
369
|
+
end
|
|
370
|
+
end
|
|
371
|
+
|
|
372
|
+
private
|
|
373
|
+
|
|
374
|
+
def test_certificate
|
|
375
|
+
private_key = OpenSSL::PKey::RSA.new(2048)
|
|
376
|
+
certificate = OpenSSL::X509::Certificate.new
|
|
377
|
+
certificate.version = 2
|
|
378
|
+
certificate.serial = 1
|
|
379
|
+
certificate.subject = OpenSSL::X509::Name.parse("/CN=site.example")
|
|
380
|
+
certificate.issuer = certificate.subject
|
|
381
|
+
certificate.public_key = private_key.public_key
|
|
382
|
+
certificate.not_before = Time.now - 60
|
|
383
|
+
certificate.not_after = Time.now + 3600
|
|
384
|
+
factory = OpenSSL::X509::ExtensionFactory.new
|
|
385
|
+
factory.subject_certificate = certificate
|
|
386
|
+
factory.issuer_certificate = certificate
|
|
387
|
+
certificate.add_extension(factory.create_extension("basicConstraints", "CA:TRUE", true))
|
|
388
|
+
certificate.add_extension(factory.create_extension("subjectAltName", "DNS:site.example"))
|
|
389
|
+
certificate.sign(private_key, OpenSSL::Digest.new("SHA256"))
|
|
390
|
+
[certificate, private_key]
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
def gzip(body)
|
|
394
|
+
output = StringIO.new
|
|
395
|
+
writer = Zlib::GzipWriter.new(output)
|
|
396
|
+
writer.write(body)
|
|
397
|
+
writer.close
|
|
398
|
+
output.string
|
|
399
|
+
end
|
|
400
|
+
|
|
401
|
+
def with_pinned_connection(local_url, resolver)
|
|
402
|
+
socket_open = TCPSocket.method(:open)
|
|
403
|
+
connections = []
|
|
404
|
+
local_port = URI.parse(local_url).port
|
|
405
|
+
with_replaced_method(Resolv, :getaddresses, resolver) do
|
|
406
|
+
replacement = lambda do |address, port, *_arguments|
|
|
407
|
+
connections << [address, port]
|
|
408
|
+
socket_open.call("127.0.0.1", local_port)
|
|
409
|
+
end
|
|
410
|
+
with_replaced_method(TCPSocket, :open, replacement) { yield connections }
|
|
411
|
+
end
|
|
412
|
+
end
|
|
413
|
+
end
|
|
@@ -98,4 +98,20 @@ class EnumerableTest < Test::Unit::TestCase
|
|
|
98
98
|
assert_equal "Has Date", latest.first[:title]
|
|
99
99
|
assert_equal "No Date", latest.last[:title]
|
|
100
100
|
end
|
|
101
|
+
|
|
102
|
+
def test_latest_preserves_invalid_dates_without_failing
|
|
103
|
+
feed = SimpleRSS.parse <<~XML
|
|
104
|
+
<rss version="2.0">
|
|
105
|
+
<channel>
|
|
106
|
+
<title>Mixed dates</title>
|
|
107
|
+
<item><title>Invalid</title><pubDate>not-a-date</pubDate></item>
|
|
108
|
+
<item><title>Valid</title><pubDate>2026-09-01T00:00:00Z</pubDate></item>
|
|
109
|
+
</channel>
|
|
110
|
+
</rss>
|
|
111
|
+
XML
|
|
112
|
+
|
|
113
|
+
assert_equal %w[Valid Invalid], feed.latest.map(&:title)
|
|
114
|
+
assert_equal "not-a-date", feed.first.pubDate
|
|
115
|
+
assert_equal %w[Invalid Valid], feed.map(&:title)
|
|
116
|
+
end
|
|
101
117
|
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
require "test_helper"
|
|
2
|
+
require "feedbag"
|
|
3
|
+
require_relative "../support/http_server"
|
|
4
|
+
|
|
5
|
+
class FeedbagIntegrationTest < Test::Unit::TestCase
|
|
6
|
+
include HTTPServer
|
|
7
|
+
|
|
8
|
+
def test_feedbag_candidates_can_be_consumed_by_simple_rss
|
|
9
|
+
html = '<html><head><link rel="alternate" type="application/rss+xml" href="/feed.xml"></head></html>'
|
|
10
|
+
xml = '<rss version="2.0"><channel><title>Example</title><item><title>Post</title></item></channel></rss>'
|
|
11
|
+
responses = [[200, { "Content-Type" => "text/html" }, html], [200, { "Content-Type" => "application/rss+xml" }, xml]]
|
|
12
|
+
with_server(responses) do |url, requests|
|
|
13
|
+
candidates = Feedbag.find(url, open_timeout: 1, read_timeout: 1)
|
|
14
|
+
assert_equal ["#{url}/feed.xml"], candidates
|
|
15
|
+
assert_equal 1, requests.size
|
|
16
|
+
feed = SimpleRSS.fetch(candidates.first, timeout: 1)
|
|
17
|
+
assert_equal "Post", feed.normalized_entries.first.title
|
|
18
|
+
assert_equal ["GET / HTTP/1.1", "GET /feed.xml HTTP/1.1"], requests.map(&:first)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|