simple-rss 2.2.0 → 2.3.0

data/test/base/category_parsing_test.rb

@@ -0,0 +1,217 @@
+require "test_helper"
+
+class CategoryParsingTest < Test::Unit::TestCase
+  def test_atom_terms_are_collected_and_filterable
+    source = <<~XML
+      <feed xmlns="http://www.w3.org/2005/Atom">
+        <title>Categories</title>
+        <entry>
+          <title>Ruby news</title>
+          <category term="ruby" label="Ruby Language"/>
+          <category term="rails" label="Rails"/>
+        </entry>
+      </feed>
+    XML
+    feed = SimpleRSS.parse(source, array_tags: [:category])
+
+    assert_equal %w[ruby rails], feed.first.category
+    assert_equal [feed.first], feed.items_by_category("ruby")
+    assert_equal "ruby", SimpleRSS.parse(source).first.category
+  end
+
+  def test_atom_attribute_variants_preserve_order_duplicates_and_source
+    source = File.read(File.join(__dir__, "../data/atom_categories.xml"))
+    feed = SimpleRSS.parse(source, array_tags: [:category])
+
+    assert_equal ["ruby", "rails", "Ruby & RSS", "ruby"], feed.first.category
+    assert_equal [feed.first], feed.items_by_category("RAILS")
+    assert_equal [], feed.items_by_category("Human label")
+    assert_nil feed[1].category
+    assert_equal ["sports"], feed[2].category
+    assert_equal source, feed.source
+    assert_equal "ruby", SimpleRSS.parse(source).first.category
+  end
+
+  def test_atom_scalar_mode_skips_missing_and_blank_terms
+    feed = parse_entry(<<~XML)
+      <category label="Label only">Not a term</category>
+      <category term=" &#32; "/>
+      <category term="first"/>
+      <category term="second"/>
+    XML
+
+    assert_equal "first", feed.first.category
+    assert_equal [feed.first], feed.items_by_category("first")
+    assert_equal [], feed.items_by_category("second")
+  end
+
+  def test_categories_exclude_nested_metadata_content_comments_and_cdata
+    feed = parse_entry(<<~XML, array_tags: [:category])
+      <source xml:base="https://example.com/"><category term="source"/></source>
+      <content type="xhtml"><div><category term="content"/></div></content>
+      <extension><category term="extension"/></extension>
+      <!-- <category term="comment"/> -->
+      <![CDATA[<category term="cdata"/>]]>
+      <?example <category term="instruction"/> ?>
+      <category term="direct"><category term="nested"/></category>
+    XML
+
+    assert_equal ["direct"], feed.first.category
+  end
+
+  def test_categories_do_not_borrow_terms_from_neighboring_elements_or_entries
+    source = <<~XML
+      <feed xmlns="http://www.w3.org/2005/Atom">
+        <title>Boundaries</title>
+        <category term="feed"/>
+        <entry>
+          <title>First</title>
+          <category label="Missing"><extension term="nested"/></category>
+          <extension term="neighbor"/>
+        </entry>
+        <entry><title>Second</title><category term="second"/></entry>
+      </feed>
+    XML
+
+    [SimpleRSS.parse(source), SimpleRSS.parse(source, array_tags: [:category])].each do |feed|
+      assert_nil feed.first.category
+      assert_equal [], feed.items_by_category("neighbor")
+      assert_equal [feed[1]], feed.items_by_category("second")
+    end
+  end
+
+  def test_namespace_prefixes_resolve_at_feed_entry_and_category_scope
+    source = <<~XML
+      <feed xmlns:atom="http://www.w3.org/2005/Atom" xmlns:Topic="http://www.w3.org/2005/Atom">
+        <title>Namespace scopes</title>
+        <atom:entry xmlns:local="http://www.w3.org/2005/Atom">
+          <title>First</title>
+          <Topic:category term="feed-scope"/>
+          <local:category term="entry-scope"/>
+          <inline:category xmlns:inline="http://www.w3.org/2005/At&#111;m" term="category-scope"/>
+          <category xmlns="http://www.w3.org/2005/Atom" term="local-default"/>
+          <category term="unqualified">Not Atom</category>
+        </atom:entry>
+      </feed>
+    XML
+    feed = SimpleRSS.parse(source, array_tags: [:category])
+
+    assert_equal %w[feed-scope entry-scope category-scope local-default], feed.first.category
+  end
+
+  def test_namespace_rebinding_and_default_resets_do_not_leak_categories
+    source = <<~XML
+      <feed xmlns="http://www.w3.org/2005/Atom" xmlns:topic="http://www.w3.org/2005/Atom">
+        <title>Rebinding</title>
+        <entry xmlns:topic="urn:other">
+          <title>First</title>
+          <topic:category term="foreign"/>
+          <category xmlns="urn:other" term="other">Foreign text</category>
+          <category xmlns="" term="reset">Unqualified text</category>
+          <topic:category xmlns:topic="http://www.w3.org/2005/Atom" term="restored"/>
+        </entry>
+        <entry><title>Second</title><topic:category term="inherited"/></entry>
+      </feed>
+    XML
+    feed = SimpleRSS.parse(source, array_tags: [:category])
+
+    assert_equal ["restored"], feed.first.category
+    assert_equal ["inherited"], feed[1].category
+  end
+
+  def test_exact_element_and_attribute_names_are_required
+    feed = parse_entry(<<~XML, array_tags: [:category])
+      <categoryish term="wrong-element"/>
+      <category data-term="wrong-attribute"/>
+      <category label="term='quoted'"/>
+      <category xmlns:other="urn:other" other:term="wrong-namespace"/>
+      <unknown:category term="undeclared"/>
+      <category term="correct"/>
+    XML
+
+    assert_equal ["correct"], feed.first.category
+  end
+
+  def test_rss_text_cdata_scalar_and_array_filters_remain_compatible
+    source = File.read(File.join(__dir__, "../data/rss_categories.xml"))
+    scalar_feed = SimpleRSS.parse(source)
+    array_feed = SimpleRSS.parse(source, array_tags: [:category])
+
+    assert_equal "Technology", scalar_feed.first.category
+    assert_equal ["Technology", "Ruby & RSS", "Technology"], array_feed.first.category
+    assert_equal [scalar_feed.first], scalar_feed.items_by_category("tech")
+    assert_equal [array_feed.first], array_feed.items_by_category("ruby")
+    assert_equal ["Sports"], array_feed[1].category
+  end
+
+  def test_rss_categories_exclude_embedded_markup_and_accept_atom_extensions
+    source = <<~XML
+      <rss version="2.0" xmlns:topic="http://www.w3.org/2005/Atom">
+        <channel>
+          <title>Mixed categories</title>
+          <item>
+            <title>First</title>
+            <description><![CDATA[<category>Embedded</category>]]></description>
+            <extension><category>Nested</category></extension>
+            <category term="ignored">RSS text</category>
+            <topic:category term="Atom term"/>
+            <category xmlns="urn:other">Foreign text</category>
+          </item>
+        </channel>
+      </rss>
+    XML
+    feed = SimpleRSS.parse(source, array_tags: [:category])
+
+    assert_equal ["RSS text", "Atom term"], feed.first.category
+  end
+
+  def test_rss_namespaces_and_empty_category_behavior_are_preserved
+    source = <<~XML
+      <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/">
+        <channel><title>RSS 1.0</title></channel>
+        <item><title>First</title><category>Ruby</category><category></category></item>
+        <item><title>Second</title><category/></item>
+      </rdf:RDF>
+    XML
+    feed = SimpleRSS.parse(source, array_tags: [:category])
+
+    assert_equal ["Ruby", ""], feed.first.category
+    assert_nil feed[1].category
+    assert_equal "", SimpleRSS.parse(source)[1].category
+  end
+
+  def test_namespace_declarations_in_comments_do_not_override_the_feed
+    source = <<~XML
+      <!-- <feed xmlns="urn:other"> -->
+      <feed xmlns="http://www.w3.org/2005/Atom">
+        <title>Actual feed</title>
+        <entry><title>First</title><category term="actual"/></entry>
+      </feed>
+    XML
+
+    assert_equal "actual", SimpleRSS.parse(source).first.category
+  end
+
+  def test_rss_channel_namespaces_do_not_apply_to_sibling_rdf_items
+    source = <<~XML
+      <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/">
+        <channel xmlns:topic="http://www.w3.org/2005/Atom"><title>RSS</title></channel>
+        <item><title>First</title><topic:category term="out-of-scope"/><category>RSS text</category></item>
+      </rdf:RDF>
+    XML
+
+    assert_equal ["RSS text"], SimpleRSS.parse(source, array_tags: [:category]).first.category
+  end
+
+  private
+
+  def parse_entry(content, options = {})
+    source = <<~XML
+      <feed xmlns="http://www.w3.org/2005/Atom">
+        <title>Categories</title>
+        <entry><title>Entry</title>#{content}</entry>
+      </feed>
+    XML
+    SimpleRSS.parse(source, options)
+  end
+end

data/test/base/date_ordering_test.rb

@@ -0,0 +1,95 @@
+require "test_helper"
+
+class DateOrderingTest < Test::Unit::TestCase
+  def setup
+    @feed = SimpleRSS.parse(File.read(File.join(__dir__, "../data/mixed_dates.xml")))
+  end
+
+  def test_latest_uses_valid_fallbacks_and_preserves_equal_date_order
+    assert_equal [
+      "Updated first", "Updated second", "Fractional", "Published", "Publication",
+      "Historical", "Missing first", "Blank", "Invalid"
+    ], @feed.latest.map(&:title)
+  end
+
+  def test_latest_limits_results_without_changing_original_entries
+    original = @feed.to_hash
+
+    assert_equal ["Updated first", "Updated second", "Fractional"], @feed.latest(3).map(&:title)
+    assert_equal [], @feed.latest(0)
+    assert_same @feed.items[3], @feed.latest.first
+    assert_equal original, @feed.to_hash
+  end
+
+  def test_latest_sorts_atom_entries_with_only_published_dates
+    feed = SimpleRSS.parse <<~XML
+      <feed xmlns="http://www.w3.org/2005/Atom">
+        <title>Publication dates</title>
+        <entry><title>Newer</title><published>2026-09-03T00:00:00Z</published></entry>
+        <entry><title>Older</title><published>2026-09-01T00:00:00Z</published></entry>
+      </feed>
+    XML
+
+    assert_equal %w[Newer Older], feed.latest.map(&:title)
+  end
+
+  def test_items_since_uses_valid_fallbacks_and_excludes_undated_entries
+    assert_equal ["Fractional", "Updated first", "Published", "Updated second"],
+                 @feed.items_since(Time.iso8601("2026-09-01T00:00:00Z")).map(&:title)
+    assert_equal ["Fractional", "Updated first", "Updated second"],
+                 @feed.items_since(Time.iso8601("2026-09-02T00:00:00Z")).map(&:title)
+    assert_equal [], @feed.items_since(Time.iso8601("2026-09-04T00:00:00Z"))
+  end
+
+  def test_merge_sorts_dated_entries_before_undated_identified_entries
+    assert_equal [
+      "Updated first", "Updated second", "Fractional", "Published", "Publication",
+      "Historical", "Blank", "Invalid", "Missing first"
+    ], @feed.merge.map(&:title)
+  end
+
+  def test_merge_uses_fallback_dates_to_dedupe_and_keeps_unidentified_entries_last
+    other_feed = SimpleRSS.parse <<~XML
+      <rss version="2.0">
+        <channel>
+          <title>Other feed</title>
+          <item>
+            <guid>updated-first</guid>
+            <title>Replacement</title>
+            <pubDate>not-a-date</pubDate>
+            <published>2026-09-04T00:00:00Z</published>
+          </item>
+          <item><title>Unidentified newest</title><pubDate>2030-01-01T00:00:00Z</pubDate></item>
+        </channel>
+      </rss>
+    XML
+    original = @feed.to_hash
+    other_original = other_feed.to_hash
+
+    assert_equal [
+      "Replacement", "Updated second", "Fractional", "Published", "Publication",
+      "Historical", "Blank", "Invalid", "Missing first", "Unidentified newest"
+    ], SimpleRSS.merge(@feed, other_feed).map(&:title)
+    assert_equal original, @feed.to_hash
+    assert_equal other_original, other_feed.to_hash
+  end
+
+  def test_merge_keeps_the_first_duplicate_when_dates_are_equal
+    other_feed = SimpleRSS.parse <<~XML
+      <rss version="2.0">
+        <channel>
+          <title>Other feed</title>
+          <item>
+            <guid>updated-second</guid>
+            <title>Later duplicate</title>
+            <updated>2026-09-03T00:00:00.000000002Z</updated>
+          </item>
+        </channel>
+      </rss>
+    XML
+
+    duplicates = @feed.merge(other_feed).select { |item| item[:guid] == "updated-second" }
+
+    assert_equal ["Updated second"], duplicates.map(&:title)
+  end
+end

data/test/base/discovery_dependency_test.rb

@@ -0,0 +1,29 @@
+require "test_helper"
+require "open3"
+
+class DiscoveryDependencyTest < Test::Unit::TestCase
+  def test_core_parsing_works_without_loading_nokogiri_and_discovery_explains_the_dependency
+    script = <<~RUBY_SCRIPT
+      require "simple-rss"
+      abort "Nokogiri loaded by core" if defined?(Nokogiri)
+      Kernel.prepend(Module.new do
+        def require(path)
+          raise LoadError, "optional dependency is unavailable" if path == "nokogiri"
+
+          super
+        end
+      end)
+      feed = SimpleRSS.parse('<rss version="2.0"><channel><title>Example</title></channel></rss>')
+      abort "Core parsing failed" unless feed.title == "Example"
+      begin
+        SimpleRSS.discover("http://127.0.0.1/", timeout: 0.1)
+        abort "Expected a dependency error"
+      rescue SimpleRSS::DiscoveryDependencyError => error
+        puts error.message
+      end
+    RUBY_SCRIPT
+    output, status = Open3.capture2e(RbConfig.ruby, "-Ilib", "-e", script)
+    assert status.success?, output
+    assert_include output, 'Install the optional "nokogiri" gem'
+  end
+end

data/test/base/discovery_test.rb

@@ -0,0 +1,151 @@
+require "test_helper"
+require_relative "../support/http_server"
+
+class DiscoveryTest < Test::Unit::TestCase
+  include HTTPServer
+
+  def test_discovers_advertised_feeds_without_fetching_them
+    html = '<html><head><link rel="alternate" type="application/rss+xml" title="News" href="/feed.xml"></head></html>'
+    with_server([[200, { "Content-Type" => "text/html" }, html]]) do |url, requests|
+      candidates = SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1)
+      assert_equal [{ url: "#{url}/feed.xml", title: "News", format: :rss, media_type: "application/rss+xml", source: :html_link, verified: false }], candidates
+      assert_equal 1, requests.size
+    end
+  end
+
+  def test_acceptance_corpus_preserves_formats_order_titles_and_distinct_queries
+    html = File.read(File.join(__dir__, "../data/discovery.html"))
+    with_server([[200, { "Content-Type" => "text/html; charset=utf-8" }, html]]) do |url, requests|
+      candidates = SimpleRSS.discover("#{url}/blog/index.html", network_policy: :unrestricted, timeout: 1)
+      assert_equal(["news.xml?edition=1&lang=en", "atom.xml", "feed.json", "legacy.json", "feed.rdf", "news.xml?edition=2&lang=en"], candidates.map { |candidate| candidate[:url].delete_prefix("#{url}/syndication/") })
+      assert_equal(%i[rss atom json_feed json_feed rss rss], candidates.map { |candidate| candidate[:format] })
+      assert_equal "News & updates", candidates.first[:title]
+      assert_equal "application/rss+xml", candidates.first[:media_type]
+      assert_equal [false], candidates.map { |candidate| candidate[:verified] }.uniq
+      assert_equal [:html_link], candidates.map { |candidate| candidate[:source] }.uniq
+      assert_equal 1, requests.size
+    end
+  end
+
+  def test_direct_empty_feeds_are_verified_without_instance_valid
+    sources = {
+      rss: '<rss version="2.0"><channel><title>RSS</title></channel></rss>',
+      atom: '<feed xmlns="http://www.w3.org/2005/Atom"><title>Atom</title></feed>',
+      json_feed: '{"version":"https://jsonfeed.org/version/1.1","title":"JSON Feed","items":[]}'
+    }
+    sources.each do |format, body|
+      with_server([[200, { "Content-Type" => "text/plain" }, body]]) do |url, requests|
+        candidate = SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1).first
+        assert_equal "#{url}/", candidate[:url]
+        assert_equal format, candidate[:format]
+        assert_equal :document, candidate[:source]
+        assert_equal true, candidate[:verified]
+        assert_equal 1, requests.size
+      end
+    end
+  end
+
+  def test_empty_self_closing_feed_containers_are_recognized
+    ['<rss version="2.0"><channel/></rss>', '<feed xmlns="http://www.w3.org/2005/Atom"/>'].each do |body|
+      with_server([[200, {}, body]]) do |url, _requests|
+        candidates = SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1)
+        assert_equal true, candidates.first[:verified]
+        assert_empty SimpleRSS.parse(body).normalized_entries
+      end
+    end
+  end
+
+  def test_rdf_feed_and_xhtml_metadata_are_supported
+    rdf = '<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/"><channel><title>RDF</title></channel></rdf:RDF>'
+    with_server([[200, {}, rdf]]) do |url, _requests|
+      candidate = SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1).first
+      assert_equal :rss, candidate[:format]
+      assert_equal "RDF", candidate[:title]
+    end
+    xhtml = '<html xmlns="http://www.w3.org/1999/xhtml"><head><link rel="alternate" type="application/atom+xml" href="feed.xml" /></head><body /></html>'
+    with_server([[200, { "Content-Type" => "application/xhtml+xml" }, xhtml]]) do |url, _requests|
+      assert_equal(["#{url}/feed.xml"], SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1).map { |candidate| candidate[:url] })
+    end
+  end
+
+  def test_redirects_base_urls_and_protocol_relative_links_preserve_queries
+    html = '<html><head><base href="../feeds/"><link rel=alternate type=application/rss+xml href="news.xml?edition=1#items"><link rel=alternate type=application/atom+xml href="//example.com/atom?q=1#entry"></head></html>'
+    responses = [[302, { "Location" => "../pages/index?view=1#head" }, ""], [200, { "Content-Type" => "text/html" }, html]]
+    with_server(responses) do |url, requests|
+      candidates = SimpleRSS.discover("#{url}/start/page", network_policy: :unrestricted, timeout: 1)
+      assert_equal(["#{url}/feeds/news.xml?edition=1", "http://example.com/atom?q=1"], candidates.map { |candidate| candidate[:url] })
+      assert_equal ["GET /start/page HTTP/1.1", "GET /pages/index?view=1 HTTP/1.1"], requests.map(&:first)
+    end
+  end
+
+  def test_invalid_links_are_ignored_and_invalid_first_base_uses_document_url
+    html = '<html><head><base href="javascript:wrong"><base href="https://wrong.example.com/">' \
+           '<link rel=alternate type=application/rss+xml href="valid.xml">' \
+           '<link rel=alternate type=application/rss+xml href="javascript:wrong">' \
+           '<link rel=alternate type=application/rss+xml href="file:///tmp/feed.xml">' \
+           '<link rel=alternate type=application/rss+xml href="http://user:password@example.com/feed">' \
+           '<link rel=alternate type=application/rss+xml href="http://[broken">' \
+           '<link rel=alternate type=application/rss+xml href=" "></head></html>'
+    with_server([[200, { "Content-Type" => "text/html" }, html]]) do |url, requests|
+      assert_equal(["#{url}/valid.xml"], SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1).map { |candidate| candidate[:url] })
+      assert_equal 1, requests.size
+    end
+  end
+
+  def test_omitted_head_tags_and_html_entities_are_parsed_as_html
+    html = '<!doctype html><title>Example</title><link rel=alternate type=application/rss+xml href="/feed?q=1&amp;b=2" title="Café &amp; ☀"><p>Post</p>'
+    with_server([[200, { "Content-Type" => "text/html; charset=utf-8" }, html]]) do |url, _requests|
+      candidate = SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1).first
+      assert_equal "#{url}/feed?q=1&b=2", candidate[:url]
+      assert_equal "Café & ☀", candidate[:title]
+    end
+  end
+
+  def test_no_feeds_is_distinct_from_http_and_parse_failures
+    with_server([[200, { "Content-Type" => "text/html" }, "<p>No feeds here</p>"]]) do |url, _requests|
+      assert_empty SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1)
+    end
+    with_server([[404, { "Content-Type" => "text/html" }, "<p>Not found</p>"]]) do |url, _requests|
+      error = assert_raise(SimpleRSS::HTTPError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1) }
+      assert_equal 404, error.status_code
+    end
+    ['{"version":', '{"hello":"world"}', '<rss version="2.0">broken</rss>', "not a feed", ""].each do |body|
+      with_server([[200, { "Content-Type" => "application/octet-stream" }, body]]) do |url, _requests|
+        assert_raise(SimpleRSS::DiscoveryError, body) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1) }
+      end
+    end
+  end
+
+  def test_feed_markup_in_html_scripts_does_not_become_a_direct_feed
+    html = '<html><head><script type="text/plain"><rss version="2.0"><channel><title>Wrong</title></channel></rss></script></head><body></body></html>'
+    with_server([[200, { "Content-Type" => "application/rss+xml" }, html]]) do |url, _requests|
+      assert_empty SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1)
+    end
+  end
+
+  def test_foreign_namespaces_do_not_fabricate_verified_feeds
+    ['<feed xmlns="urn:component"/>', '<rss xmlns="urn:component"><channel/></rss>'].each do |body|
+      with_server([[200, { "Content-Type" => "text/html" }, body]]) do |url, _requests|
+        assert_empty SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1)
+      end
+    end
+  end
+
+  def test_parser_limits_fail_clearly
+    html = "<html><head></head><body>" + ("<div>" * 140)
+    with_server([[200, { "Content-Type" => "text/html" }, html]]) do |url, _requests|
+      assert_raise(SimpleRSS::DiscoveryError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1) }
+    end
+  end
+
+  def test_advertised_private_urls_are_only_unverified_metadata
+    html = '<html><head><link rel=alternate type=application/rss+xml href="http://127.0.0.1:9/feed"></head></html>'
+    with_server([[200, { "Content-Type" => "text/html" }, html]]) do |url, requests|
+      candidate = SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1).first
+      assert_equal false, candidate[:verified]
+      assert_equal "http://127.0.0.1:9/feed", candidate[:url]
+      assert_raise(SimpleRSS::PolicyError) { SimpleRSS.fetch(candidate[:url], network_policy: :public) }
+      assert_equal 1, requests.size
+    end
+  end
+end