RubyGems - simple-rss - Versions diffs - 2.1.0 → 2.3.0 - Mend

simple-rss 2.1.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (46) hide show

checksums.yaml +4 -4
data/CHANGELOG.md +174 -0
data/README.md +492 -8
data/Rakefile +1 -1
data/examples/digest.rb +19 -0
data/examples/discover.rb +14 -0
data/examples/feedbag.rb +10 -0
data/lib/simple-rss/discovery.rb +107 -0
data/lib/simple-rss/entry_normalizer.rb +356 -0
data/lib/simple-rss/http_client.rb +216 -0
data/lib/simple-rss/json_entry_normalizer.rb +147 -0
data/lib/simple-rss/json_feed.rb +176 -0
data/lib/simple-rss/normalized_entry.rb +79 -0
data/lib/simple-rss/request_errors.rb +21 -0
data/lib/simple-rss/request_policy.rb +72 -0
data/lib/simple-rss/xml_element.rb +136 -0
data/lib/simple-rss.rb +437 -71
data/simple-rss.gemspec +5 -5
data/test/base/category_parsing_test.rb +217 -0
data/test/base/date_ordering_test.rb +95 -0
data/test/base/discovery_dependency_test.rb +29 -0
data/test/base/discovery_test.rb +151 -0
data/test/base/discovery_transport_test.rb +413 -0
data/test/base/enumerable_test.rb +16 -0
data/test/base/feed_merging_and_diffing_test.rb +140 -0
data/test/base/feedbag_integration_test.rb +21 -0
data/test/base/fetch_integration_test.rb +25 -0
data/test/base/fetch_test.rb +0 -27
data/test/base/filtering_and_validation_test.rb +187 -0
data/test/base/json_feed_test.rb +372 -0
data/test/base/media_and_enclosure_helpers_test.rb +84 -0
data/test/base/normalized_entries_test.rb +410 -0
data/test/base/normalized_fetch_test.rb +132 -0
data/test/base/relation_links_test.rb +162 -0
data/test/data/atom_categories.xml +23 -0
data/test/data/atom_nested_link.xml +13 -0
data/test/data/discovery.html +28 -0
data/test/data/json_feed_1.json +75 -0
data/test/data/json_feed_1_1.json +78 -0
data/test/data/mixed_dates.xml +55 -0
data/test/data/normalized_atom.xml +22 -0
data/test/data/normalized_rss.xml +21 -0
data/test/data/rss_categories.xml +15 -0
data/test/support/http_server.rb +44 -0
data/test/support/replace_method.rb +14 -0
metadata +46 -10

data/test/base/discovery_transport_test.rb ADDED Viewed

@@ -0,0 +1,413 @@
+require "test_helper"
+require "simple-rss/http_client"
+require "zlib"
+require "stringio"
+require_relative "../support/http_server"
+require_relative "../support/replace_method"
+class DiscoveryTransportTest < Test::Unit::TestCase
+  include HTTPServer
+  include ReplaceMethod
+  PAGE = [200, { "Content-Type" => "text/html" }, "<html><head></head><body>Example</body></html>"].freeze
+  FEED = '<rss version="2.0"><channel><title>Example</title></channel></rss>'.freeze
+  def test_public_policy_rejects_prohibited_ipv4_and_ipv6_before_connecting
+    prohibited = %w[
+      0.0.0.0 10.0.0.1 100.64.0.1 127.0.0.1 169.254.169.254 172.16.0.1 192.0.0.1 192.0.2.1
+      192.88.99.1 192.168.1.1 198.18.0.1 198.51.100.1 203.0.113.1 224.0.0.1 255.255.255.255
+      [::] [::1] [::ffff:127.0.0.1] [64:ff9b::7f00:1] [100::1] [2001::1] [2001:db8::1]
+      [2002:7f00:1::] [3fff::1] [fc00::1] [fe80::1] [ff02::1]
+    ]
+    with_replaced_method(TCPSocket, :open, ->(*) { flunk "Prohibited address reached the socket" }) do
+      prohibited.each do |host|
+        assert_raise(SimpleRSS::PolicyError, host) { SimpleRSS.discover("http://#{host}/", timeout: 1) }
+        assert_raise(SimpleRSS::PolicyError, host) { SimpleRSS.discover(host, timeout: 1) }
+      end
+    end
+  end
+  def test_malformed_urls_and_schemes_fail_before_connecting
+    with_replaced_method(TCPSocket, :open, ->(*) { flunk "Invalid URL reached the socket" }) do
+      [nil, 123, [], {}, "", "/relative", "ftp://example.com/feed", "file:///tmp/feed", "mailto:reader@example.com", "javascript:alert(1)", "http://", "http://[broken", "http://example.com:0", "http://example.com:65536", "https://user:password@example.com/", "user:password@example.com"].each do |url|
+        assert_raise(SimpleRSS::PolicyError, url.inspect) { SimpleRSS.discover(url) }
+      end
+    end
+  end
+  def test_mixed_public_and_private_dns_answers_are_rejected
+    with_replaced_method(Resolv, :getaddresses, ->(_host) { ["8.8.8.8", "127.0.0.1"] }) do
+      with_replaced_method(TCPSocket, :open, ->(*) { flunk "Mixed DNS answers reached the socket" }) do
+        assert_raise(SimpleRSS::PolicyError) { SimpleRSS.discover("http://site.example/") }
+      end
+    end
+  end
+  def test_checked_dns_address_is_pinned_and_host_header_is_preserved
+    resolutions = 0
+    resolver = lambda do |host|
+      assert_equal "site.example", host
+      resolutions += 1
+      resolutions == 1 ? ["8.8.8.8"] : ["127.0.0.1"]
+    end
+    with_server([PAGE]) do |local_url, requests|
+      with_pinned_connection(local_url, resolver) do |connections|
+        assert_empty SimpleRSS.discover("http://site.example/", timeout: 1)
+        assert_equal [["8.8.8.8", 80]], connections
+        assert_equal 1, resolutions
+        assert_include requests.first, "Host: site.example"
+      end
+    end
+  end
+  def test_dns_policy_is_rechecked_on_each_redirect
+    resolutions = 0
+    resolver = lambda do |_host|
+      resolutions += 1
+      resolutions == 1 ? ["8.8.8.8"] : ["127.0.0.1"]
+    end
+    with_server([[302, { "Location" => "/next" }, ""]]) do |local_url, requests|
+      with_pinned_connection(local_url, resolver) do |connections|
+        assert_raise(SimpleRSS::PolicyError) { SimpleRSS.discover("http://site.example/", timeout: 1) }
+        assert_equal 2, resolutions
+        assert_equal 1, connections.size
+        assert_equal 1, requests.size
+      end
+    end
+  end
+  def test_redirect_to_a_private_literal_is_blocked
+    with_server([[302, { "Location" => "http://127.0.0.1:9/feed" }, ""]]) do |local_url, requests|
+      with_pinned_connection(local_url, ->(_host) { ["8.8.8.8"] }) do |connections|
+        assert_raise(SimpleRSS::PolicyError) { SimpleRSS.discover("http://site.example/", timeout: 1) }
+        assert_equal 1, connections.size
+        assert_equal 1, requests.size
+      end
+    end
+  end
+  def test_public_ipv6_addresses_are_checked_and_pinned
+    with_server([PAGE]) do |local_url, _requests|
+      with_pinned_connection(local_url, ->(_host) { ["2606:4700:4700::1111"] }) do |connections|
+        assert_empty SimpleRSS.discover("http://site.example/", timeout: 1)
+        assert_equal [["2606:4700:4700::1111", 80]], connections
+      end
+    end
+  end
+  def test_an_application_policy_can_allow_one_internal_destination
+    with_server([PAGE]) do |url, requests|
+      policy = ->(uri, address) { uri.hostname == "127.0.0.1" && IPAddr.new("127.0.0.1/32").include?(address) }
+      assert_empty SimpleRSS.discover(url, network_policy: policy, timeout: 1)
+      assert_equal 1, requests.size
+    end
+  end
+  def test_cross_origin_redirects_strip_credentials_and_keep_safe_headers
+    headers = { "Authorization" => "Bearer fixture", "Cookie" => "session=fixture", "X-Api-Key" => "fixture", "User-Agent" => "Feed tests", "Accept" => "text/html", "Accept-Language" => "en" }
+    options = { network_policy: :unrestricted, timeout: 1, headers: headers, etag: '"fixture"', last_modified: "Sat, 12 Sep 2026 10:00:00 GMT" }
+    with_server([PAGE]) do |destination, destination_requests|
+      with_server([[302, { "Location" => destination }, ""]]) do |source, source_requests|
+        assert_empty SimpleRSS.discover(source, options)
+        assert_include source_requests.first, "Authorization: Bearer fixture"
+        assert_include source_requests.first, "Cookie: session=fixture"
+        assert_include source_requests.first, 'If-None-Match: "fixture"'
+        assert_empty destination_requests.first.grep(/Authorization:|Cookie:|Api-Key:|If-None-Match:|If-Modified-Since:/i)
+        assert_include destination_requests.first, "User-Agent: Feed tests"
+        assert_include destination_requests.first, "Accept: text/html"
+        assert_include destination_requests.first, "Accept-Language: en"
+        assert_equal "Bearer fixture", options[:headers]["Authorization"]
+        assert_equal '"fixture"', options[:etag]
+      end
+    end
+  end
+  def test_same_origin_redirects_keep_authorization
+    with_server([[302, { "Location" => "/next" }, ""], PAGE]) do |url, requests|
+      SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1, headers: { "Authorization" => "Bearer fixture" })
+      requests.each { |request| assert_include request, "Authorization: Bearer fixture" }
+    end
+  end
+  def test_redirect_loops_and_limits_are_bounded
+    with_server([[302, { "Location" => "/" }, ""]]) do |url, requests|
+      assert_raise(SimpleRSS::RedirectError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1) }
+      assert_equal 1, requests.size
+    end
+    with_server([[302, { "Location" => "/next" }, ""]]) do |url, requests|
+      assert_raise(SimpleRSS::RedirectError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1, max_redirects: 0) }
+      assert_equal 1, requests.size
+    end
+    ["file:///tmp/feed", "ftp://example.com/feed", "http://[invalid", "https://user:password@example.com/"].each do |location|
+      with_server([[302, { "Location" => location }, ""]]) do |url, requests|
+        assert_raise(SimpleRSS::PolicyError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1) }
+        assert_equal 1, requests.size
+      end
+    end
+  end
+  def test_bodyless_conditional_responses_ignore_representation_encoding
+    response = [304, { "Content-Encoding" => "gzip", "ETag" => '"fixture"' }, ""]
+    with_server([response, response]) do |url, _requests|
+      assert_nil SimpleRSS.fetch(url, network_policy: :unrestricted, timeout: 1, max_bytes: 1, etag: '"fixture"')
+      error = assert_raise(SimpleRSS::HTTPError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1) }
+      assert_equal 304, error.status_code
+    end
+  end
+  def test_truncated_content_length_is_a_transport_failure
+    response = lambda do |client, _request|
+      client.write("HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: #{PAGE.last.bytesize + 20}\r\nConnection: close\r\n\r\n#{PAGE.last}")
+    end
+    with_server([response]) do |url, _requests|
+      assert_raise(SimpleRSS::RequestError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1) }
+    end
+  end
+  def test_no_follow_option_reports_the_redirect_status
+    with_server([[302, { "Location" => "/next" }, ""]]) do |url, requests|
+      error = assert_raise(SimpleRSS::HTTPError) { SimpleRSS.discover(url, network_policy: :unrestricted, follow_redirects: false, timeout: 1) }
+      assert_equal 302, error.status_code
+      assert_equal 1, requests.size
+    end
+  end
+  def test_response_byte_limit_accepts_the_boundary_and_rejects_overflow
+    with_server([PAGE, PAGE]) do |url, requests|
+      assert_empty SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1, max_bytes: PAGE.last.bytesize)
+      assert_raise(SimpleRSS::ResponseTooLarge) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1, max_bytes: PAGE.last.bytesize - 1) }
+      assert_equal 2, requests.size
+    end
+  end
+  def test_chunked_body_is_stopped_before_its_end
+    response = lambda do |client, _request|
+      client.write("HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nTransfer-Encoding: chunked\r\n\r\n100\r\n" + ("x" * 256) + "\r\n")
+      wait_for_disconnect(client)
+    end
+    with_server([response]) do |url, requests|
+      assert_raise(SimpleRSS::ResponseTooLarge) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1, max_bytes: 128) }
+      assert_equal 1, requests.size
+    end
+  end
+  def test_gzip_and_deflate_are_decoded_with_a_decompressed_size_limit
+    { "gzip" => gzip(PAGE.last), "deflate" => Zlib::Deflate.deflate(PAGE.last) }.each do |encoding, compressed|
+      with_server([[200, PAGE[1].merge("Content-Encoding" => encoding), compressed]]) do |url, _requests|
+        assert_empty SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1)
+      end
+    end
+    compressed = gzip("<html><body>" + ("x" * 200_000) + "</body></html>")
+    assert_operator compressed.bytesize, :<, 1024
+    with_server([[200, PAGE[1].merge("Content-Encoding" => "gzip"), compressed]]) do |url, _requests|
+      assert_raise(SimpleRSS::ResponseTooLarge) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1, max_bytes: 1024) }
+    end
+  end
+  def test_body_budget_is_shared_across_redirects
+    responses = [[302, { "Location" => "/next" }, "x" * 100], PAGE]
+    with_server(responses) do |url, requests|
+      assert_raise(SimpleRSS::ResponseTooLarge) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1, max_bytes: 100 + PAGE.last.bytesize - 1) }
+      assert_equal 2, requests.size
+    end
+  end
+  def test_invalid_compression_and_partial_responses_fail_clearly
+    responses = [
+      [200, PAGE[1].merge("Content-Encoding" => "br"), "wrong"],
+      [200, PAGE[1].merge("Content-Encoding" => "gzip"), "wrong"],
+      [200, PAGE[1].merge("Content-Encoding" => "gzip"), gzip(PAGE.last).byteslice(0, 15)],
+      [200, PAGE[1].merge("Content-Encoding" => "gzip"), gzip(PAGE.last) + gzip("ignored")],
+      [206, PAGE[1].merge("Content-Range" => "bytes 0-9/100"), "0123456789"]
+    ]
+    responses.each do |response|
+      with_server([response]) do |url, _requests|
+        assert_raise(SimpleRSS::RequestError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1) }
+      end
+    end
+  end
+  def test_read_timeout_is_visible_and_does_not_retry
+    with_server([->(client, _request) { wait_for_disconnect(client) }]) do |url, requests|
+      assert_raise(SimpleRSS::RequestTimeout) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 0.1) }
+      assert_equal 1, requests.size
+    end
+  end
+  def test_dns_resolution_is_inside_the_total_timeout
+    with_replaced_method(Resolv, :getaddresses, ->(_host) { Queue.new.pop }) do
+      assert_raise(SimpleRSS::RequestTimeout) { SimpleRSS.discover("http://site.example/", timeout: 0.1) }
+    end
+  end
+  def test_missing_dns_results_and_connection_failures_are_not_empty_candidates
+    with_replaced_method(Resolv, :getaddresses, ->(_host) { [] }) do
+      assert_raise(SimpleRSS::RequestError) { SimpleRSS.discover("http://site.example/") }
+    end
+    with_replaced_method(Resolv, :getaddresses, ->(_host) { ["8.8.8.8"] }) do
+      with_replaced_method(TCPSocket, :open, ->(*) { raise Errno::ECONNREFUSED }) do
+        assert_raise(SimpleRSS::RequestError) { SimpleRSS.discover("http://site.example/") }
+      end
+    end
+  end
+  def test_policy_disables_environment_proxies_and_keeps_tls_verification
+    constructor = Net::HTTP.method(:new)
+    sessions = []
+    replacement = lambda do |*arguments|
+      assert_equal ["site.example", 443, nil], arguments
+      session = constructor.call(*arguments)
+      sessions << session
+      session
+    end
+    with_replaced_method(Resolv, :getaddresses, ->(_host) { ["8.8.8.8"] }) do
+      with_replaced_method(Net::HTTP, :new, replacement) do
+        with_replaced_method(TCPSocket, :open, ->(*) { raise Errno::ECONNREFUSED }) do
+          assert_raise(SimpleRSS::RequestError) { SimpleRSS.discover("https://site.example/", timeout: 1) }
+        end
+      end
+    end
+    assert_equal false, sessions.first.proxy?
+    assert_equal "site.example", sessions.first.address
+    assert_equal "8.8.8.8", sessions.first.ipaddr
+    assert_equal OpenSSL::SSL::VERIFY_PEER, sessions.first.verify_mode
+    assert_equal true, sessions.first.verify_hostname
+  end
+  def test_pinned_https_preserves_certificate_hostname_verification
+    certificate, private_key = test_certificate
+    store = OpenSSL::X509::Store.new
+    store.add_cert(certificate)
+    context = OpenSSL::SSL::SSLContext.new
+    context.cert = certificate
+    context.key = private_key
+    server = TCPServer.new("127.0.0.1", 0)
+    ssl_server = OpenSSL::SSL::SSLServer.new(server, context)
+    requests = []
+    worker = Thread.new do
+      5.times do
+        client = nil
+        begin
+          client = ssl_server.accept
+          request = []
+          while (line = client.gets)
+            break if line == "\r\n"
+            request << line.strip
+          end
+          requests << request
+          client.write("HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: #{PAGE.last.bytesize}\r\nConnection: close\r\n\r\n#{PAGE.last}")
+        rescue OpenSSL::SSL::SSLError
+          next
+        ensure
+          client&.close
+        end
+      end
+    end
+    constructor = Net::HTTP.method(:new)
+    replacement = lambda do |*arguments|
+      session = constructor.call(*arguments)
+      session.cert_store = store
+      session
+    end
+    with_replaced_method(Net::HTTP, :new, replacement) do
+      with_pinned_connection("http://127.0.0.1:#{server.addr[1]}", ->(_host) { ["8.8.8.8"] }) do |connections|
+        assert_empty SimpleRSS.discover("https://site.example/", timeout: 1)
+        assert_empty SimpleRSS.discover("site.example", timeout: 1)
+        assert_empty SimpleRSS.discover("site.example/blog?edition=1#entries", timeout: 1)
+        assert_empty SimpleRSS.discover("//site.example/blog", timeout: 1)
+        error = assert_raise(SimpleRSS::RequestError) { SimpleRSS.discover("https://wrong.example/", timeout: 1) }
+        assert_kind_of OpenSSL::SSL::SSLError, error.cause
+        assert_equal Array.new(5) { ["8.8.8.8", 443] }, connections
+      end
+    end
+    worker.value
+    assert_equal ["GET / HTTP/1.1", "GET / HTTP/1.1", "GET /blog?edition=1 HTTP/1.1", "GET /blog HTTP/1.1"], requests.map(&:first)
+    assert_include requests.first, "Host: site.example"
+  ensure
+    worker&.kill
+    worker&.join
+    server&.close
+  end
+  def test_bounded_fetch_reuses_the_policy_and_conditional_get
+    headers = { "ETag" => '"fixture"', "Last-Modified" => "Sat, 12 Sep 2026 10:00:00 GMT" }
+    with_server([[200, headers, FEED], [304, headers, ""]]) do |url, requests|
+      feed = SimpleRSS.fetch(url, network_policy: :unrestricted, timeout: 1)
+      assert_equal "Example", feed.title
+      assert_equal "#{url}/", feed.source_url
+      assert_nil SimpleRSS.fetch(url, network_policy: :unrestricted, timeout: 1, etag: feed.etag, last_modified: feed.last_modified)
+      assert_include requests.last, 'If-None-Match: "fixture"'
+      assert_include requests.last, "If-Modified-Since: #{headers["Last-Modified"]}"
+    end
+  end
+  def test_bodyless_success_responses_raise_feed_errors
+    with_server([[204, {}, ""]]) do |url, _requests|
+      assert_raise(SimpleRSS::DiscoveryError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1) }
+    end
+    [{}, { network_policy: :unrestricted }].each do |options|
+      with_server([[204, {}, ""]]) do |url, _requests|
+        error = assert_raise(SimpleRSSError) { SimpleRSS.fetch(url, options.merge(timeout: 1)) }
+        assert_equal "Poorly formatted feed", error.message
+      end
+    end
+  end
+  def test_invalid_options_and_controlled_headers_fail_before_connecting
+    options = [
+      { network_policy: nil }, { network_policy: :unknown }, { timeout: 0 }, { timeout: nil }, { timeout: Float::INFINITY },
+      { max_bytes: 0 }, { max_bytes: 1.5 }, { max_redirects: -1 }, { max_redirects: 1.5 }, { headers: [] },
+      { headers: { "X-Value" => "bad\r\nInjected: header" } }, { headers: { "Bad Header" => "value" } }
+    ]
+    options += %w[Host Proxy-Authorization Accept-Encoding Range Connection Transfer-Encoding].map { |name| { headers: { name => "value" } } }
+    with_replaced_method(TCPSocket, :open, ->(*) { flunk "Invalid options reached the socket" }) do
+      options.each do |override|
+        assert_raise(ArgumentError, override.inspect) { SimpleRSS.discover("https://site.example/", override) }
+      end
+    end
+  end
+  private
+  def test_certificate
+    private_key = OpenSSL::PKey::RSA.new(2048)
+    certificate = OpenSSL::X509::Certificate.new
+    certificate.version = 2
+    certificate.serial = 1
+    certificate.subject = OpenSSL::X509::Name.parse("/CN=site.example")
+    certificate.issuer = certificate.subject
+    certificate.public_key = private_key.public_key
+    certificate.not_before = Time.now - 60
+    certificate.not_after = Time.now + 3600
+    factory = OpenSSL::X509::ExtensionFactory.new
+    factory.subject_certificate = certificate
+    factory.issuer_certificate = certificate
+    certificate.add_extension(factory.create_extension("basicConstraints", "CA:TRUE", true))
+    certificate.add_extension(factory.create_extension("subjectAltName", "DNS:site.example"))
+    certificate.sign(private_key, OpenSSL::Digest.new("SHA256"))
+    [certificate, private_key]
+  end
+  def gzip(body)
+    output = StringIO.new
+    writer = Zlib::GzipWriter.new(output)
+    writer.write(body)
+    writer.close
+    output.string
+  end
+  def with_pinned_connection(local_url, resolver)
+    socket_open = TCPSocket.method(:open)
+    connections = []
+    local_port = URI.parse(local_url).port
+    with_replaced_method(Resolv, :getaddresses, resolver) do
+      replacement = lambda do |address, port, *_arguments|
+        connections << [address, port]
+        socket_open.call("127.0.0.1", local_port)
+      end
+      with_replaced_method(TCPSocket, :open, replacement) { yield connections }
+    end
+  end
+end

data/test/base/enumerable_test.rb CHANGED Viewed

@@ -98,4 +98,20 @@ class EnumerableTest < Test::Unit::TestCase
     assert_equal "Has Date", latest.first[:title]
     assert_equal "No Date", latest.last[:title]
   end
+  def test_latest_preserves_invalid_dates_without_failing
+    feed = SimpleRSS.parse <<~XML
+      <rss version="2.0">
+        <channel>
+          <title>Mixed dates</title>
+          <item><title>Invalid</title><pubDate>not-a-date</pubDate></item>
+          <item><title>Valid</title><pubDate>2026-09-01T00:00:00Z</pubDate></item>
+        </channel>
+      </rss>
+    XML
+    assert_equal %w[Valid Invalid], feed.latest.map(&:title)
+    assert_equal "not-a-date", feed.first.pubDate
+    assert_equal %w[Invalid Valid], feed.map(&:title)
+  end
 end

data/test/base/feed_merging_and_diffing_test.rb ADDED Viewed

@@ -0,0 +1,140 @@
+require "test_helper"
+class FeedMergingAndDiffingTest < Test::Unit::TestCase
+  def setup
+    @feed_one = SimpleRSS.parse <<~XML
+      <?xml version="1.0" encoding="UTF-8"?>
+      <rss version="2.0">
+        <channel>
+          <title>Feed One</title>
+          <item>
+            <guid>shared-guid</guid>
+            <title>Shared (older)</title>
+            <pubDate>Mon, 01 Jan 2024 10:00:00 UTC</pubDate>
+          </item>
+          <item>
+            <guid>one-guid</guid>
+            <title>Only One</title>
+            <pubDate>Mon, 01 Jan 2024 11:00:00 UTC</pubDate>
+          </item>
+          <item>
+            <title>Unidentified One</title>
+          </item>
+        </channel>
+      </rss>
+    XML
+    @feed_two = SimpleRSS.parse <<~XML
+      <?xml version="1.0" encoding="UTF-8"?>
+      <rss version="2.0">
+        <channel>
+          <title>Feed Two</title>
+          <item>
+            <guid>shared-guid</guid>
+            <title>Shared (newer)</title>
+            <pubDate>Mon, 01 Jan 2024 12:00:00 UTC</pubDate>
+          </item>
+          <item>
+            <link>https://example.com/two-only</link>
+            <title>Only Two</title>
+            <pubDate>Mon, 01 Jan 2024 13:00:00 UTC</pubDate>
+          </item>
+          <item>
+            <title>Unidentified Two</title>
+          </item>
+        </channel>
+      </rss>
+    XML
+  end
+  def test_merge_dedupes_and_sorts_newest_first
+    merged = @feed_one.merge(@feed_two)
+    titles = merged.map { |item| item[:title] }
+    assert_equal [
+      "Only Two",
+      "Shared (newer)",
+      "Only One",
+      "Unidentified One",
+      "Unidentified Two"
+    ], titles
+  end
+  def test_class_merge_combines_multiple_feeds
+    merged = SimpleRSS.merge(@feed_one, @feed_two)
+    assert_equal 5, merged.size
+    assert_equal "Only Two", merged.first[:title]
+  end
+  def test_diff_reports_added_and_removed_items
+    old_feed = SimpleRSS.parse <<~XML
+      <?xml version="1.0" encoding="UTF-8"?>
+      <rss version="2.0">
+        <channel>
+          <title>Old Feed</title>
+          <item>
+            <guid>stay</guid>
+            <title>Stay</title>
+          </item>
+          <item>
+            <guid>remove</guid>
+            <title>Remove</title>
+          </item>
+        </channel>
+      </rss>
+    XML
+    new_feed = SimpleRSS.parse <<~XML
+      <?xml version="1.0" encoding="UTF-8"?>
+      <rss version="2.0">
+        <channel>
+          <title>New Feed</title>
+          <item>
+            <guid>stay</guid>
+            <title>Stay</title>
+          </item>
+          <item>
+            <guid>add</guid>
+            <title>Add</title>
+          </item>
+        </channel>
+      </rss>
+    XML
+    diff = old_feed.diff(new_feed)
+    assert_equal(["Add"], diff[:added].map { |item| item[:title] })
+    assert_equal(["Remove"], diff[:removed].map { |item| item[:title] })
+  end
+  def test_dedupe_mutates_items_and_keeps_unidentified_entries
+    feed = SimpleRSS.parse <<~XML
+      <?xml version="1.0" encoding="UTF-8"?>
+      <rss version="2.0">
+        <channel>
+          <title>Dedupe Feed</title>
+          <item>
+            <guid>duplicate</guid>
+            <title>First duplicate</title>
+          </item>
+          <item>
+            <guid>duplicate</guid>
+            <title>Second duplicate</title>
+          </item>
+          <item>
+            <title>Unidentified One</title>
+          </item>
+          <item>
+            <title>Unidentified Two</title>
+          </item>
+        </channel>
+      </rss>
+    XML
+    result = feed.dedupe
+    assert_same feed, result
+    assert_equal(["First duplicate", "Unidentified One", "Unidentified Two"], feed.items.map { |item| item[:title] })
+  end
+end

data/test/base/feedbag_integration_test.rb ADDED Viewed

@@ -0,0 +1,21 @@
+require "test_helper"
+require "feedbag"
+require_relative "../support/http_server"
+class FeedbagIntegrationTest < Test::Unit::TestCase
+  include HTTPServer
+  def test_feedbag_candidates_can_be_consumed_by_simple_rss
+    html = '<html><head><link rel="alternate" type="application/rss+xml" href="/feed.xml"></head></html>'
+    xml = '<rss version="2.0"><channel><title>Example</title><item><title>Post</title></item></channel></rss>'
+    responses = [[200, { "Content-Type" => "text/html" }, html], [200, { "Content-Type" => "application/rss+xml" }, xml]]
+    with_server(responses) do |url, requests|
+      candidates = Feedbag.find(url, open_timeout: 1, read_timeout: 1)
+      assert_equal ["#{url}/feed.xml"], candidates
+      assert_equal 1, requests.size
+      feed = SimpleRSS.fetch(candidates.first, timeout: 1)
+      assert_equal "Post", feed.normalized_entries.first.title
+      assert_equal ["GET / HTTP/1.1", "GET /feed.xml HTTP/1.1"], requests.map(&:first)
+    end
+  end
+end

data/test/base/fetch_integration_test.rb ADDED Viewed

@@ -0,0 +1,25 @@
+require "test_helper"
+# Integration tests that require network access
+# These are skipped by default, run with NETWORK_TESTS=1
+class FetchIntegrationTest < Test::Unit::TestCase
+  def test_fetch_real_feed
+    omit unless ENV["NETWORK_TESTS"]
+    rss = SimpleRSS.fetch("https://feeds.bbci.co.uk/news/rss.xml", timeout: 10)
+    assert_kind_of SimpleRSS, rss
+    assert rss.title
+    assert rss.items.any?
+  end
+  def test_fetch_stores_caching_headers
+    omit unless ENV["NETWORK_TESTS"]
+    rss = SimpleRSS.fetch("https://feeds.bbci.co.uk/news/rss.xml", timeout: 10)
+    assert(rss.etag || rss.last_modified, "Expected ETag or Last-Modified header")
+  end
+  def test_fetch_follows_redirect
+    omit unless ENV["NETWORK_TESTS"]
+    rss = SimpleRSS.fetch("https://github.com/cardmagic/simple-rss/commits/master.atom", timeout: 10)
+    assert_kind_of SimpleRSS, rss
+  end
+end

data/test/base/fetch_test.rb CHANGED Viewed

@@ -88,30 +88,3 @@ class FetchTest < Test::Unit::TestCase
     end
   end
 end
-# Integration tests that require network access
-# These are skipped by default, run with NETWORK_TESTS=1
-class FetchIntegrationTest < Test::Unit::TestCase
-  def test_fetch_real_feed
-    omit unless ENV["NETWORK_TESTS"]
-    # Use a reliable, long-lived RSS feed
-    rss = SimpleRSS.fetch("https://feeds.bbci.co.uk/news/rss.xml", timeout: 10)
-    assert_kind_of SimpleRSS, rss
-    assert rss.title
-    assert rss.items.any?
-  end
-  def test_fetch_stores_caching_headers
-    omit unless ENV["NETWORK_TESTS"]
-    rss = SimpleRSS.fetch("https://feeds.bbci.co.uk/news/rss.xml", timeout: 10)
-    # At least one of these should be present for most feeds
-    assert(rss.etag || rss.last_modified, "Expected ETag or Last-Modified header")
-  end
-  def test_fetch_follows_redirect
-    omit unless ENV["NETWORK_TESTS"]
-    # GitHub raw URLs often redirect
-    rss = SimpleRSS.fetch("https://github.com/cardmagic/simple-rss/commits/master.atom", timeout: 10)
-    assert_kind_of SimpleRSS, rss
-  end
-end