simple-rss 2.2.0 → 2.3.0

data/lib/simple-rss.rb

@@ -10,18 +10,25 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength
   # @rbs!
   #   include Enumerable[Hash[Symbol, untyped]]
 
-  VERSION = "2.2.0".freeze
+  VERSION = "2.3.0".freeze
 
   # @rbs @items: Array[Hash[Symbol, untyped]]
   # @rbs @source: String
   # @rbs @options: Hash[Symbol, untyped]
+  # @rbs @json_feed: JsonFeed?
   # @rbs @etag: String?
   # @rbs @last_modified: String?
+  # @rbs @entry_contexts: Hash[Hash[Symbol, untyped], Hash[Symbol, untyped]]
 
   attr_reader :items #: Array[Hash[Symbol, untyped]]
   attr_reader :source #: String
   attr_reader :etag #: String?
   attr_reader :last_modified #: String?
+  attr_reader :source_url #: String?
+  attr_reader :raw_json #: Hash[String, untyped]?
+  attr_reader :home_page_url, :feed_url, :favicon, :next_url, :user_comment #: String?
+  attr_reader :authors, :hubs #: untyped
+  attr_reader :expired #: bool?
   alias entries items #: Array[Hash[Symbol, untyped]]
 
   @@feed_tags = %i[
@@ -63,6 +70,11 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength
     @items = [] #: Array[Hash[Symbol, untyped]]
     @options = {} #: Hash[Symbol, untyped]
     @options.update(options)
+    @source_url = options[:source_url]
+    @json_feed = nil
+    @raw_json = nil
+    @entry_contexts = {} #: Hash[Hash[Symbol, untyped], Hash[Symbol, untyped]]
+    @entry_contexts.compare_by_identity
 
     parse
   end
@@ -73,6 +85,22 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength
   end
   alias feed channel
 
+  # @rbs (?source_url: String?, ?mappings: Hash[Symbol, untyped]) -> Array[NormalizedEntry]
+  def normalized_entries(source_url: nil, mappings: {})
+    json_feed = @json_feed
+    raise ArgumentError, "XML mappings are not supported for JSON Feed" if json_feed && !mappings.empty?
+    return items.map { |item| json_feed.normalized_entry(item, source_url: source_url || @source_url) } if json_feed
+
+    EntryNormalizer.validate_mappings(mappings)
+    feed_authors = normalized_feed_authors
+    items.map do |item|
+      context = @entry_contexts[item] || raise(SimpleRSSError, "Cannot normalize an item without its original XML source")
+      element = XmlElement.new(context[:name], context[:attributes], context[:content], context[:parent])
+      EntryNormalizer.new(element, item, source_url: source_url || @source_url, mappings: mappings,
+                                         raw_xml: context[:xml], feed_authors: feed_authors).entry
+    end
+  end
+
   # Iterate over all items in the feed
   #
   # @rbs () { (Hash[Symbol, untyped]) -> void } -> self
@@ -95,11 +123,13 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength
   #
   # @rbs (?Integer) -> Array[Hash[Symbol, untyped]]
   def latest(count = 10)
-    items.sort_by { |item| item[:pubDate] || item[:updated] || Time.at(0) }.reverse.first(count)
+    sorted_items_by_date(items).first(count)
   end
 
   # @rbs () -> Symbol
   def feed_type
+    return :json_feed if @json_feed
+
     atom_namespaced_feed = source.match?(/<(atom:)?feed\b[^>]*xmlns(:\w+)?=['"][^'"]*atom/i)
     return :atom if atom_namespaced_feed
     return :rss2 if source.match?(/<rss[^>]*version=['"]2/i)
@@ -111,6 +141,8 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength
 
   # @rbs () -> bool
   def valid?
+    return true if @json_feed
+
     return false if items.empty?
 
     title_value = instance_variable_get(:@title)
@@ -123,8 +155,8 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength
   # @rbs (Time) -> Array[Hash[Symbol, untyped]]
   def items_since(time)
     items.select do |item|
-      item_date = item[:pubDate] || item[:updated] || item[:published]
-      item_date.is_a?(Time) && item_date > time
+      date = item_date(item)
+      date && date > time
     end
   end
 
@@ -152,7 +184,8 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength
   # @rbs (*SimpleRSS) -> Array[Hash[Symbol, untyped]]
   def merge(*feeds)
     all_items = [items, *feeds.map(&:items)].flatten
-    dedupe_items(sorted_items_by_date(all_items))
+    keyed_items, unkeyed_items = all_items.partition { |item| !item_key(item).nil? }
+    dedupe_items(sorted_items_by_date(keyed_items) + unkeyed_items)
   end
 
   # @rbs (SimpleRSS) -> Hash[Symbol, Array[Hash[Symbol, untyped]]]
@@ -194,7 +227,8 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength
 
   # @rbs (?Hash[Symbol, untyped]) -> Hash[Symbol, untyped]
   def as_json(_options = {})
-    hash = {} #: Hash[Symbol, untyped]
+    raw_json = @raw_json
+    hash = raw_json ? raw_json.transform_keys(&:to_sym) : {} #: Hash[Symbol, untyped]
 
     @@feed_tags.each do |tag|
       tag_cleaned = clean_tag(tag)
@@ -219,6 +253,8 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength
 
   # @rbs (?format: Symbol) -> String
   def to_xml(format: :rss2)
+    raise SimpleRSSError, "JSON Feed to XML conversion is not supported" if @json_feed
+
     case format
     when :rss2 then to_rss2_xml
     when :atom then to_atom_xml
@@ -278,85 +314,68 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength
       require "net/http"
       require "uri"
 
-      uri = URI.parse(url)
-      response = perform_fetch(uri, options)
+      require_relative "simple-rss/http_client"
+      response, final_uri = HTTPClient.new(options).get(url)
 
       return nil if response.is_a?(Net::HTTPNotModified)
 
       raise SimpleRSSError, "HTTP #{response.code}: #{response.message}" unless response.is_a?(Net::HTTPSuccess)
 
-      body = response.body.force_encoding(Encoding::UTF_8)
-      feed = parse(body, options)
+      body = (response.body || "").force_encoding(Encoding::UTF_8)
+      feed = parse(body, options.merge(source_url: final_uri.to_s))
       feed.instance_variable_set(:@etag, response["ETag"])
       feed.instance_variable_set(:@last_modified, response["Last-Modified"])
       feed
     end
 
-    private
-
-    # @rbs (untyped, Hash[Symbol, untyped]) -> untyped
-    def perform_fetch(uri, options)
-      http = build_http(uri, options)
-      request = build_request(uri, options)
-
-      response = http.request(request)
-      handle_redirect(response, options) || response
+    # @rbs (String, ?Hash[Symbol, untyped]) -> Array[Hash[Symbol, untyped]]
+    def discover(url, options = {})
+      require_relative "simple-rss/discovery"
+      Discovery.new(options).discover(url)
     end
+  end
 
-    # @rbs (untyped, Hash[Symbol, untyped]) -> untyped
-    def build_http(uri, options)
-      host = uri.host || raise(SimpleRSSError, "Invalid URL: missing host")
-      http = Net::HTTP.new(host, uri.port)
-      http.use_ssl = uri.scheme == "https"
-
-      timeout = options[:timeout]
-      if timeout
-        http.open_timeout = timeout
-        http.read_timeout = timeout
-      end
-
-      http
-    end
+  DATE_TAGS = %i[pubDate lastBuildDate published updated expirationDate modified dc:date].freeze
+  STRIP_HTML_TAGS = %i[author contributor skipHours skipDays].freeze
+  ATOM_NAMESPACE = "http://www.w3.org/2005/Atom".freeze
+  RSS_NAMESPACES = [nil, "", "http://purl.org/rss/1.0/", "http://my.netscape.com/rdf/simple/0.9/"].freeze
+  XML_TAG_PATTERN = %r{<!--.*?-->|<!\[CDATA\[.*?\]\]>|<\?.*?\?>|<(/?)([\w:.-]+)((?:[^<>"']|"[^"]*"|'[^']*')*)>}m
 
-    # @rbs (untyped, Hash[Symbol, untyped]) -> untyped
-    def build_request(uri, options)
-      request = Net::HTTP::Get.new(uri)
-      request["User-Agent"] = "SimpleRSS/#{VERSION}"
+  private
 
-      # Conditional GET headers
-      request["If-None-Match"] = options[:etag] if options[:etag]
-      request["If-Modified-Since"] = options[:last_modified] if options[:last_modified]
+  # @rbs () -> Array[XmlElement]
+  def normalized_feed_authors
+    document = XmlElement.new("", "", @source, { namespaces: {}, base_urls: [] })
+    feed = document.children.find { |element| element.matches?("feed", ATOM_NAMESPACE) }
+    return [] unless feed
 
-      # Custom headers
-      options[:headers]&.each { |key, value| request[key] = value }
+    feed.children.select { |element| element.matches?("author", ATOM_NAMESPACE) }
+  end
 
-      request
+  # @rbs () -> void
+  def parse
+    prefix = @source.b.sub(/\A\xEF\xBB\xBF/n, "").lstrip
+    return parse_xml unless prefix.match?(/\A(?:[\{\["0-9-]|true\b|false\b|null\b)/n)
+
+    json_feed = JsonFeed.new(@source)
+    @json_feed = json_feed
+    @raw_json = json_feed.document
+    JsonFeed::FEED_FIELDS.each do |field|
+      instance_variable_set("@#{field}", json_feed.document[field])
+      self.class.attr_reader(field)
     end
-
-    # @rbs (untyped, Hash[Symbol, untyped]) -> untyped
-    def handle_redirect(response, options)
-      return nil unless response.is_a?(Net::HTTPRedirection)
-      return nil if options[:follow_redirects] == false
-
-      location = response["Location"]
-      return nil unless location
-
-      redirects = (options[:_redirects] || 0) + 1
-      raise SimpleRSSError, "Too many redirects" if redirects > 5
-
-      new_options = options.merge(_redirects: redirects)
-      perform_fetch(URI.parse(location), new_options)
+    @link = json_feed.document["home_page_url"]
+    self.class.attr_reader(:link)
+    @items = json_feed.items
+    @items.each do |item|
+      item.define_singleton_method(:method_missing) { |name, *_args| self[name] }
+      add_item_media_helpers(item)
     end
   end
 
-  DATE_TAGS = %i[pubDate lastBuildDate published updated expirationDate modified dc:date].freeze
-  STRIP_HTML_TAGS = %i[author contributor skipHours skipDays].freeze
-
-  private
-
   # @rbs () -> void
-  def parse
-    raise SimpleRSSError, "Poorly formatted feed" unless @source =~ %r{<(channel|feed).*?>.*?</(channel|feed)>}mi
+  def parse_xml
+    raise SimpleRSSError, "Poorly formatted feed" unless @source =~ %r{<(channel|feed).*?>.*?</(channel|feed)>|<(channel|feed)\b[^>]*?/\s*>}mi
 
     # Feed's title and link
     feed_content = Regexp.last_match(1) if @source =~ %r{(.*?)<(rss:|atom:)?(item|entry).*?>.*?</(rss:|atom:)?(item|entry)>}mi
@@ -396,12 +415,29 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength
     end
 
     # RSS items' title, link, and description
-    @source.scan(%r{<(rss:|atom:)?(item|entry)([\s][^>]*)?>(.*?)</(rss:|atom:)?(item|entry)>}mi) do |match|
+    namespace_contexts = entry_contexts
+    entry_pattern = %r{<(rss:|atom:)?(item|entry)([\s][^>]*)?>(.*?)</(rss:|atom:)?(item|entry)>}mi
+    position = 0
+    while (match = entry_pattern.match(@source, position))
+      position = match.end(0)
       item = {} #: Hash[Symbol, untyped]
+      parent_context = namespace_contexts[match.begin(0)]
+      namespaces = parent_context && parent_context[:namespaces].merge(namespace_attributes(match[3].to_s))
+      @entry_contexts[item] = {
+        name: "#{match[1]}#{match[2]}", attributes: match[3].to_s, content: match[4].to_s,
+        parent: parent_context || { namespaces: {}, base_urls: [] }, xml: match[0]
+      }
       @@item_tags.each do |tag|
         next if tag.to_s.strip.empty?
 
-        parse_item_tag(item, tag, match[3], match[2])
+        if tag == :category
+          next unless namespaces
+
+          parse_category_tag(item, match[4].to_s, namespaces, element_namespace("#{match[1]}#{match[2]}", namespaces))
+          next
+        end
+
+        parse_item_tag(item, tag, match[4], match[3])
       end
       item.define_singleton_method(:method_missing) { |name, *_args| self[name] }
       add_item_media_helpers(item)
@@ -444,15 +480,120 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength
 
   # @rbs (Hash[Symbol, untyped], String, String) -> void
   def parse_rel_tag(item, tag_str, content)
-    tag, rel = tag_str.split("+")
+    tag, rel = tag_str.split("+", 2)
     return unless tag && rel
 
-    content =~ %r{<(rss:|atom:)?#{tag}(.*?)rel=['"]#{rel}['"](.*?)>(.*?)</(rss:|atom:)?#{tag}>}mi ||
-      content =~ %r{<(rss:|atom:)?#{tag}(.*?)rel=['"]#{rel}['"](.*?)/\s*>}mi
+    value = if tag == "link"
+              link_relation_href(content, rel)
+            else
+              content =~ %r{<(rss:|atom:)?#{tag}(.*?)rel=['"]#{rel}['"](.*?)>(.*?)</(rss:|atom:)?#{tag}>}mi ||
+                content =~ %r{<(rss:|atom:)?#{tag}(.*?)rel=['"]#{rel}['"](.*?)/\s*>}mi
+
+              return unless Regexp.last_match(3) || Regexp.last_match(4)
+
+              clean_content(tag.to_sym, Regexp.last_match(3), Regexp.last_match(4))
+            end
+    return if value.nil?
+
+    item[clean_tag("#{tag}+#{rel}")] = value
+    item[clean_tag("#{tag}_#{rel}")] = value
+  end
+
+  # @rbs (String, String) -> String?
+  def link_relation_href(content, relation)
+    attributes = entry_link_attributes(content).find { |link| link["rel"]&.casecmp?(relation) }
+    return unless attributes
+
+    attributes["href"]
+  end
+
+  # @rbs (String) -> Array[Hash[String, String]]
+  def entry_link_attributes(content)
+    child_elements(content).filter_map do |tag, attributes, _body|
+      next unless %w[link atom:link rss:link].include?(tag.downcase)
 
-    return unless Regexp.last_match(3) || Regexp.last_match(4)
+      xml_attributes(attributes).transform_keys(&:downcase)
+    end
+  end
 
-    item[clean_tag("#{tag}+#{rel}")] = clean_content(tag.to_sym, Regexp.last_match(3), Regexp.last_match(4))
+  # @rbs (String) -> Array[[String, String, String?]]
+  def child_elements(content)
+    XmlElement.child_elements(content)
+  end
+
+  # @rbs (Hash[Symbol, untyped], String, Hash[String, String], String?) -> void
+  def parse_category_tag(item, content, namespaces, entry_namespace)
+    values = child_elements(content).filter_map do |tag, raw_attributes, body|
+      next unless tag.split(":").last&.casecmp?("category")
+
+      attributes = xml_attributes(raw_attributes)
+      namespace = element_namespace(tag, namespaces.merge(namespace_attributes(raw_attributes)))
+      if namespace == ATOM_NAMESPACE
+        term = CGI.unescapeHTML(attributes["term"].to_s).strip
+        next term unless term.empty?
+
+        next
+      end
+
+      next if entry_namespace == ATOM_NAMESPACE || !RSS_NAMESPACES.include?(namespace)
+      next if tag.include?(":") && namespace.nil?
+      next if body.nil? && (array_tag?(:category) || !raw_attributes.rstrip.end_with?("/"))
+
+      unescape(body.to_s)
+    end
+    return if values.empty?
+
+    item[:category] = array_tag?(:category) ? values : values.first
+  end
+
+  # @rbs () -> Hash[Integer, Hash[Symbol, untyped]]
+  def entry_contexts
+    contexts = {} #: Hash[Integer, Hash[Symbol, untyped]]
+    scopes = [{ namespaces: {}, base_urls: [] }] #: Array[Hash[Symbol, untyped]]
+    position = 0
+
+    while (token = XML_TAG_PATTERN.match(@source, position))
+      position = token.end(0)
+      attributes = token[3]
+      next unless attributes
+
+      if token[1] == "/"
+        scopes.pop if scopes.size > 1
+        next
+      end
+
+      parent = scopes.fetch(-1)
+      tag = token[2].to_s.split(":").last
+      contexts[token.begin(0).to_i] = parent if %w[item entry].include?(tag&.downcase)
+      base_url = xml_attributes(attributes)["xml:base"]
+      base_urls = parent[:base_urls].dup
+      base_urls << CGI.unescapeHTML(base_url) if base_url
+      context = {
+        namespaces: parent[:namespaces].merge(namespace_attributes(attributes)),
+        base_urls: base_urls
+      }
+      scopes << context unless attributes.rstrip.end_with?("/")
+    end
+
+    contexts
+  end
+
+  # @rbs (String) -> Hash[String, String]
+  def namespace_attributes(attributes)
+    xml_attributes(attributes)
+      .select { |name, _value| name == "xmlns" || name.start_with?("xmlns:") }
+      .transform_values { |value| CGI.unescapeHTML(value) }
+  end
+
+  # @rbs (String, Hash[String, String]) -> String?
+  def element_namespace(tag, namespaces)
+    key = tag.include?(":") ? "xmlns:#{tag.split(":").first}" : "xmlns"
+    namespaces[key]
+  end
+
+  # @rbs (String) -> Hash[String, String]
+  def xml_attributes(attributes)
+    XmlElement.attributes(attributes)
   end
 
   # @rbs (String, String?) -> void
@@ -579,8 +720,10 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength
 
   # @rbs (Array[Hash[Symbol, untyped]]) -> Array[Hash[Symbol, untyped]]
   def sorted_items_by_date(item_list)
-    keyed_items, unkeyed_items = item_list.partition { |item| !item_key(item).nil? }
-    keyed_items.sort_by { |item| item_date(item) || Time.at(0) }.reverse + unkeyed_items
+    item_list.sort_by.with_index do |item, index|
+      date = item_date(item)
+      [date ? -date.to_r : Float::INFINITY, index]
+    end
   end
 
   # @rbs (Array[Hash[Symbol, untyped]]) -> Array[Hash[Symbol, untyped]]
@@ -612,10 +755,7 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength
 
   # @rbs (Hash[Symbol, untyped]) -> Time?
   def item_date(item)
-    date = item[:pubDate] || item[:updated] || item[:published]
-    return date if date.is_a?(Time)
-
-    nil
+    [item[:pubDate], item[:updated], item[:published]].find { |date| date.is_a?(Time) }
   end
 
   # @rbs (Hash[Symbol, untyped]) -> Array[String]
@@ -763,5 +903,13 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength
   end
 end
 
+require_relative "simple-rss/xml_element"
+require_relative "simple-rss/normalized_entry"
+require_relative "simple-rss/json_feed"
+require_relative "simple-rss/json_entry_normalizer"
+require_relative "simple-rss/entry_normalizer"
+
 class SimpleRSSError < StandardError # rubocop:disable Style/OneClassPerFile
 end
+
+require_relative "simple-rss/request_errors"

data/simple-rss.gemspec

@@ -1,13 +1,13 @@
 Gem::Specification.new do |s|
   s.name = "simple-rss"
-  s.version = "2.2.0"
-  s.date = "2025-12-28"
-  s.summary = "A simple, flexible, extensible, and liberal RSS and Atom reader for Ruby. It is designed to be backwards compatible with the standard RSS parser, but will never do RSS generation."
+  s.version = "2.3.0"
+  s.summary = "A flexible RSS, Atom, and JSON Feed reader for Ruby."
   s.email = "lucas@rufy.com"
   s.homepage = "https://github.com/cardmagic/simple-rss"
-  s.description = "A simple, flexible, extensible, and liberal RSS and Atom reader for Ruby. It is designed to be backwards compatible with the standard RSS parser, but will never do RSS generation."
+  s.metadata["changelog_uri"] = "https://github.com/cardmagic/simple-rss/blob/master/CHANGELOG.md"
+  s.description = "Parse RSS, Atom, and JSON Feed with normalized entries, HTTP fetching, website feed discovery, and JSON/XML serialization."
   s.authors = ["Lucas Carlson"]
-  s.files = Dir["lib/**/*", "test/**/*", "LICENSE", "README.md", "Rakefile", "simple-rss.gemspec"]
+  s.files = Dir["lib/**/*", "examples/**/*", "test/**/*", "LICENSE", "README.md", "CHANGELOG.md", "Rakefile", "simple-rss.gemspec"]
   s.required_ruby_version = ">= 3.1"
   s.add_development_dependency "rake"
   s.add_development_dependency "rdoc"