simple-rss 2.2.0 → 2.3.0

data/lib/simple-rss/discovery.rb

@@ -0,0 +1,107 @@
+# rbs_inline: enabled
+
+require_relative "http_client"
+
+class SimpleRSS::Discovery
+  MEDIA_TYPES = {
+    "application/rss+xml" => :rss, "application/rdf+xml" => :rss,
+    "application/atom+xml" => :atom, "application/feed+json" => :json_feed, "application/json" => :json_feed
+  }.freeze
+
+  # @rbs @options: Hash[Symbol, untyped]
+  # @rbs @parser: untyped
+
+  # @rbs (Hash[Symbol, untyped]) -> void
+  def initialize(options)
+    @options = { network_policy: :public }.merge(options)
+    require "nokogiri"
+    @parser = Object.const_get(:Nokogiri)
+    unless @parser.const_defined?(:HTML5)
+      raise SimpleRSS::DiscoveryDependencyError, "Discovery requires Nokogiri HTML5 support (available on CRuby)"
+    end
+  rescue LoadError
+    raise SimpleRSS::DiscoveryDependencyError, 'Install the optional "nokogiri" gem (>= 1.16, < 2) to use SimpleRSS.discover'
+  end
+
+  # @rbs (String) -> Array[Hash[Symbol, untyped]]
+  def discover(url)
+    raise SimpleRSS::PolicyError, "Expected a website URL string" unless url.is_a?(String)
+
+    url = "https:#{url}" if url.start_with?("//")
+    url = "https://#{url}" unless url.match?(/\A[a-z][a-z\d+.-]*:/i)
+    response, uri = SimpleRSS::HTTPClient.new(@options).get(url)
+    raise SimpleRSS::HTTPError, response.code.to_i unless response.is_a?(Net::HTTPSuccess)
+
+    candidates(response.body.to_s, uri, response.content_type, response.type_params["charset"])
+  end
+
+  private
+
+  # @rbs (String, untyped, String?, String?) -> Array[Hash[Symbol, untyped]]
+  def candidates(body, uri, media_type, encoding)
+    prefix = body.b.sub(/\A\xEF\xBB\xBF/n, "").lstrip
+    return [feed_candidate(body, uri, :json_feed)] if prefix.start_with?("{", "[")
+
+    document = @parser::XML.parse(body, uri.to_s, nil, @parser::XML::ParseOptions::NONET | @parser::XML::ParseOptions::RECOVER)
+    root = document.root
+    format = root && xml_format(root)
+    return [feed_candidate(body, uri, format)] if format
+
+    unless root&.name&.casecmp?("html") || %w[text/html application/xhtml+xml].include?(media_type) || prefix.match?(/\A(?:<!doctype\s+html|<html\b|<head\b)/i)
+      raise SimpleRSS::DiscoveryError, "Response is not a recognized feed or HTML page"
+    end
+
+    html_candidates(body, uri, encoding)
+  rescue SimpleRSS::RequestError, SimpleRSS::DiscoveryError
+    raise
+  rescue SimpleRSSError, ArgumentError, EncodingError => e
+    raise SimpleRSS::DiscoveryError, "Cannot parse discovery response: #{e.message}"
+  end
+
+  # @rbs (untyped) -> Symbol?
+  def xml_format(root)
+    return :rss if root.name == "rss" && root.namespace.nil?
+    return :atom if root.name == "feed" && [SimpleRSS::ATOM_NAMESPACE, "http://purl.org/atom/ns#"].include?(root.namespace&.href)
+    return :rss if root.name == "RDF" && root.namespace&.href == "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+
+    nil
+  end
+
+  # @rbs (String, untyped, Symbol) -> Hash[Symbol, untyped]
+  def feed_candidate(body, uri, format)
+    feed = SimpleRSS.parse(body, source_url: uri.to_s)
+    title = feed.instance_variable_get(:@title)
+    { url: uri.to_s, title: title, format: format, media_type: MEDIA_TYPES.key(format), source: :document, verified: true }
+  end
+
+  # @rbs (String, untyped, String?) -> Array[Hash[Symbol, untyped]]
+  def html_candidates(body, uri, encoding)
+    document = @parser::HTML5.parse(body, uri.to_s, encoding, max_tree_depth: 128, max_attributes: 128)
+    head = document.at_css("html > head")
+    return [] unless head
+
+    base = resolve_url(head.at_xpath("./base[@href]")&.[]("href"), uri) || uri
+    candidates = head.xpath("./link[@rel][@type][@href]").filter_map do |link|
+      next unless link["rel"].downcase.split.include?("alternate")
+
+      media_type = link["type"].split(";", 2).first.to_s.strip.downcase
+      format = MEDIA_TYPES[media_type]
+      next unless format
+
+      target = resolve_url(link["href"], base)
+      next unless target
+
+      { url: target.to_s, title: link["title"], format: format, media_type: media_type, source: :html_link, verified: false }
+    end
+    candidates.uniq { |candidate| candidate[:url] }
+  end
+
+  # @rbs (String?, untyped) -> untyped
+  def resolve_url(value, base)
+    return if value.nil? || value.strip.empty?
+
+    SimpleRSS::RequestPolicy.parse_url(URI.join(base.to_s, value.strip).to_s)
+  rescue URI::Error, SimpleRSS::PolicyError
+    nil
+  end
+end

data/lib/simple-rss/entry_normalizer.rb

@@ -0,0 +1,356 @@
+# rbs_inline: enabled
+
+require "uri"
+
+class SimpleRSS::EntryNormalizer
+  ATOM = SimpleRSS::ATOM_NAMESPACE
+  CONTENT = "http://purl.org/rss/1.0/modules/content/".freeze
+  DUBLIN_CORE = "http://purl.org/dc/elements/1.1/".freeze
+  MEDIA = "http://search.yahoo.com/mrss/".freeze
+  ITUNES = "http://www.itunes.com/dtds/podcast-1.0.dtd".freeze
+  XHTML = "http://www.w3.org/1999/xhtml".freeze
+
+  # @rbs (Hash[Symbol, untyped]) -> void
+  def self.validate_mappings(mappings)
+    allowed = %i[content_html content_text categories]
+    raise ArgumentError, "mappings must be a Hash with keys #{allowed.join(", ")}" unless mappings.is_a?(Hash) && (mappings.keys - allowed).empty?
+
+    %i[content_html content_text].each do |field|
+      next unless mappings.key?(field)
+      raise ArgumentError, "#{field} mapping must be a nonempty XML tag name" unless mappings[field].is_a?(String) && !mappings[field].strip.empty?
+    end
+    return unless mappings.key?(:categories)
+
+    categories = mappings[:categories]
+    raise ArgumentError, "categories mapping must be an array of tag/separator records" unless categories.is_a?(Array)
+
+    categories.each do |mapping|
+      valid = mapping.is_a?(Hash) && (mapping.keys - %i[tag separator]).empty? && mapping[:tag].is_a?(String) && !mapping[:tag].strip.empty?
+      valid &&= !mapping.key?(:separator) || (mapping[:separator].is_a?(String) && !mapping[:separator].empty?)
+      raise ArgumentError, "category mappings require a tag and an optional nonempty separator" unless valid
+    end
+  end
+
+  # @rbs (SimpleRSS::XmlElement, Hash[Symbol, untyped], Hash[Symbol, untyped]) -> void
+  def initialize(element, raw, options)
+    @element = element
+    @children = element.children
+    @source_url = options[:source_url] #: String?
+    @mappings = options.fetch(:mappings) #: Hash[Symbol, untyped]
+    @feed_authors = options.fetch(:feed_authors) #: Array[SimpleRSS::XmlElement]
+    @values = { raw: raw, raw_xml: options[:raw_xml], field_sources: {}, issues: [] } #: Hash[Symbol, untyped]
+    @field_elements = {} #: Hash[Symbol, SimpleRSS::XmlElement]
+  end
+
+  # @rbs () -> SimpleRSS::NormalizedEntry
+  def entry
+    read_identity
+    read_links
+    read_dates
+    read_content
+    read_categories
+    read_attachments
+    read_authors
+    SimpleRSS::NormalizedEntry.new(@values)
+  end
+
+  private
+
+  # @rbs () -> bool
+  def atom?
+    @element.namespace == ATOM
+  end
+
+  # @rbs (String) -> Array[SimpleRSS::XmlElement]
+  def core_elements(name)
+    @children.select { |element| atom? ? element.matches?(name, ATOM) : element.rss?(name) }
+  end
+
+  # @rbs (String, String) -> Array[SimpleRSS::XmlElement]
+  def extension_elements(name, namespace)
+    @children.select { |element| element.matches?(name, namespace) }
+  end
+
+  # @rbs (Symbol, untyped, SimpleRSS::XmlElement) -> void
+  def assign(field, value, element)
+    return if value.nil? || value == ""
+
+    @values[field] = value
+    @values[:field_sources][field] = element.name
+    @field_elements[field] = element
+  end
+
+  # @rbs (Symbol, Symbol, untyped, SimpleRSS::XmlElement) -> nil
+  def issue(field, code, value, element)
+    @values[:issues] << { field: field, code: code, value: value, source: element.name }
+    nil
+  end
+
+  # @rbs () -> void
+  def read_identity
+    identifier = core_elements(atom? ? "id" : "guid").find { |element| !element.text.empty? }
+    assign(:identifier, identifier.text, identifier) if identifier
+    title = core_elements("title").first
+    assign(:title, title.text, title) if title
+  end
+
+  # @rbs () -> void
+  def read_links
+    elements = @children.select { |element| element.matches?("link", ATOM) || (!atom? && element.rss?("link")) }
+    links = elements.map do |element|
+      href = element.namespace == ATOM ? element.attributes["href"] : element.text
+      relation = element.namespace == ATOM ? element.attributes.fetch("rel", "alternate") : "alternate"
+      relation = relation.delete_prefix("http://www.iana.org/assignments/relation/")
+      { url: resolve_url(href, element, :url), rel: relation, media_type: element.attributes["type"], raw: element.raw }
+    end
+    @values[:links] = links
+    candidates = links.each_index.select { |index| links[index][:rel] == "alternate" && links[index][:url] }
+    selected = candidates.min_by do |index|
+      link = links[index]
+      media_type = link[:media_type].to_s.split(";").first.to_s.downcase.strip
+      [!atom? && elements.fetch(index).rss?("link") ? 0 : 1, media_type_priority(media_type), index]
+    end
+    assign(:url, links[selected][:url], elements.fetch(selected)) if selected
+  end
+
+  # @rbs (String) -> Integer
+  def media_type_priority(media_type)
+    return 0 if %w[text/html application/xhtml+xml].include?(media_type)
+    return 1 if media_type.empty?
+
+    2
+  end
+
+  # @rbs (String?, SimpleRSS::XmlElement, Symbol) -> String?
+  def resolve_url(value, element, field)
+    return if value.nil? || value.strip.empty?
+
+    reference = value.strip
+    return reference if URI.parse(reference).absolute?
+
+    base = base_url(element)
+    unless base
+      issue(field, :relative_url_without_base, value, element)
+      return reference
+    end
+    URI.join(base, reference).to_s
+  rescue URI::Error
+    issue(field, :invalid_url, value, element)
+    value
+  end
+
+  # @rbs (SimpleRSS::XmlElement) -> String?
+  def base_url(element)
+    candidates = [@source_url, *element.base_urls].compact
+    start = candidates.rindex { |value| URI.parse(value).absolute? }
+    return unless start
+
+    base = candidates.fetch(start)
+    candidates.drop(start + 1).each { |value| base = URI.join(base, value).to_s }
+    base
+  rescue URI::Error
+    issue(:base_url, :invalid_url, candidates, element)
+  end
+
+  # @rbs () -> void
+  def read_dates
+    published = atom? ? core_elements("published") : core_elements("pubDate") + extension_elements("date", DUBLIN_CORE)
+    updated = extension_elements("updated", ATOM)
+    updated += core_elements("modified") unless atom?
+    read_date(:published_at, published)
+    read_date(:updated_at, updated)
+  end
+
+  # @rbs (Symbol, Array[SimpleRSS::XmlElement]) -> void
+  def read_date(field, elements)
+    elements.each do |element|
+      next if element.text.empty?
+
+      begin
+        assign(field, Time.parse(element.text), element)
+        break
+      rescue ArgumentError, RangeError
+        issue(field, :invalid_date, element.text, element)
+      end
+    end
+  end
+
+  # @rbs () -> void
+  def read_content
+    %i[content_html content_text].each do |field|
+      selector = @mappings[field]
+      element = selector && @children.find { |child| child.selected?(selector) && !child.text.empty? }
+      assign(field, element.text, element) if element
+    end
+    encoded = extension_elements("encoded", CONTENT).find { |element| !element.text.empty? }
+    assign(:content_html, encoded.text, encoded) if encoded && !@values[:content_html]
+    read_atom_content
+    full_content = @field_elements[:content_html] || @field_elements[:content_text]
+    @values[:content_base_url] = base_url(content_container(full_content)) if full_content
+    summary = core_elements(atom? ? "summary" : "description").first
+    return unless summary
+
+    value, type = atom? ? text_construct(summary, :summary) : [summary.text, :html]
+    assign(:summary, value, summary)
+    @values[:summary_type] = type if @values[:summary]
+  end
+
+  # @rbs () -> void
+  def read_atom_content
+    content = extension_elements("content", ATOM).first
+    return unless content
+    return assign(:content_url, resolve_url(content.attributes["src"], content, :content_url), content) if content.attributes["src"]
+
+    value, type = text_construct(content, :content)
+    field = type == :html ? :content_html : :content_text
+    assign(field, value, content) unless @values[field]
+  end
+
+  # @rbs (SimpleRSS::XmlElement) -> SimpleRSS::XmlElement
+  def content_container(element)
+    type = element.attributes["type"].to_s.split(";", 2).first.to_s.strip.downcase
+    return element unless %w[xhtml application/xhtml+xml].include?(type)
+
+    element.children.find { |child| child.matches?("div", XHTML) } || element
+  end
+
+  # @rbs (SimpleRSS::XmlElement, Symbol) -> [String?, Symbol?]
+  def text_construct(element, field)
+    type = element.attributes.fetch("type", "text").split(";", 2).first.to_s.strip.downcase
+    case type
+    when "text", "text/plain"
+      return [element.text, :text] if element.children.empty?
+
+      issue(field, :unexpected_markup, element.content, element)
+    when "html", "text/html"
+      return [element.text, :html]
+    when "xhtml", "application/xhtml+xml"
+      children = element.children
+      return [children.first.xhtml_content, :html] if children.size == 1 && children.first.matches?("div", XHTML)
+
+      issue(field, :invalid_xhtml, element.content, element)
+    else
+      issue(field, :unsupported_content_type, type, element)
+    end
+    [nil, nil]
+  end
+
+  # @rbs () -> void
+  def read_categories
+    details = @children.flat_map do |element|
+      mappings = @mappings[:categories] || [] #: Array[Hash[Symbol, untyped]]
+      mapping = mappings.find { |candidate| element.selected?(candidate[:tag]) }
+      term = category_term(element)
+      next [] unless mapping || term
+
+      terms = [term]
+      terms = mapping[:separator] ? element.text.split(mapping[:separator]) : [element.text] if mapping
+      terms.filter_map do |value|
+        next if value.nil? || value.strip.empty?
+
+        {
+          term: value.strip, label: element.attributes["label"], scheme: element.attributes["scheme"] || element.attributes["domain"],
+          source: element.name, raw: element.raw
+        }
+      end
+    end
+    @values[:category_details] = details
+    @values[:categories] = details.map { |category| category[:term] }.uniq
+  end
+
+  # @rbs (SimpleRSS::XmlElement) -> String?
+  def category_term(element)
+    return element.attributes["term"] if element.matches?("category", ATOM)
+    return element.text if !atom? && element.rss?("category")
+    return element.text if element.matches?("subject", DUBLIN_CORE)
+    return element.text if element.matches?("keywords", MEDIA) || element.matches?("keywords", ITUNES)
+
+    nil
+  end
+
+  # @rbs () -> void
+  def read_attachments
+    elements = @children.flat_map do |element|
+      next [element] unless element.matches?("group", MEDIA)
+
+      element.children.select { |child| child.matches?("content", MEDIA) }
+    end
+    attachments = elements.filter_map do |element|
+      attributes = element.attributes
+      if element.matches?("content", MEDIA)
+        attachment(element, attributes["url"], attributes["fileSize"], attributes["duration"])
+      elsif !atom? && element.rss?("enclosure")
+        attachment(element, attributes["url"], attributes["length"], nil)
+      elsif element.matches?("link", ATOM) && attributes["rel"].to_s.delete_prefix("http://www.iana.org/assignments/relation/") == "enclosure"
+        attachment(element, attributes["href"], attributes["length"], nil)
+      end
+    end
+    duration = extension_elements("duration", ITUNES).first
+    if attachments.size == 1 && duration && !attachments.first[:raw][:attributes].key?("duration")
+      attachments.first[:duration_in_seconds] = duration_seconds(duration.text, duration)
+      attachments.first[:raw_duration] = duration.raw
+    end
+    @values[:attachments] = attachments
+  end
+
+  # @rbs (SimpleRSS::XmlElement, String?, String?, String?) -> Hash[Symbol, untyped]?
+  def attachment(element, url, size, duration)
+    resolved_url = resolve_url(url, element, :attachment_url)
+    return unless resolved_url
+
+    {
+      url: resolved_url, media_type: element.attributes["type"], size_in_bytes: size_in_bytes(size, element),
+      duration_in_seconds: duration_seconds(duration, element), source: element.name, raw: element.raw
+    }
+  end
+
+  # @rbs (String?, SimpleRSS::XmlElement) -> Integer?
+  def size_in_bytes(value, element)
+    return if value.nil?
+    return value.to_i if value.match?(/\A\d+\z/)
+
+    issue(:size_in_bytes, :invalid_number, value, element)
+  end
+
+  # @rbs (String?, SimpleRSS::XmlElement) -> (Integer | Float)?
+  def duration_seconds(value, element)
+    return if value.nil?
+    return value.to_f if value.match?(/\A\d+(?:\.\d+)?\z/) && value.to_f.finite?
+
+    unless element.matches?("duration", ITUNES) && value.match?(/\A\d+:\d{2}(?::\d{2})?\z/)
+      return issue(:duration_in_seconds, :invalid_number, value, element)
+    end
+
+    parts = value.split(":").map(&:to_i)
+    return issue(:duration_in_seconds, :invalid_number, value, element) unless parts.drop(1).all? { |part| part < 60 }
+
+    parts.reduce(0) { |total, part| (total * 60) + part }
+  end
+
+  # @rbs () -> void
+  def read_authors
+    @values[:authors] = author_elements.map do |element|
+      next { name: element.text, email: nil, url: nil, raw: element.raw } if element.matches?("creator", DUBLIN_CORE)
+      next { name: nil, email: element.text, url: nil, raw: element.raw } unless element.namespace == ATOM
+
+      children = element.children
+      name = children.find { |child| child.matches?("name", ATOM) }
+      email = children.find { |child| child.matches?("email", ATOM) }
+      uri = children.find { |child| child.matches?("uri", ATOM) }
+      { name: name&.text, email: email&.text, url: uri && resolve_url(uri.text, uri, :author_url), raw: element.raw }
+    end
+  end
+
+  # @rbs () -> Array[SimpleRSS::XmlElement]
+  def author_elements
+    elements = core_elements("author")
+    return elements + extension_elements("creator", DUBLIN_CORE) unless atom?
+    return elements unless elements.empty?
+
+    source = core_elements("source").first
+    return @feed_authors unless source
+
+    elements = source.children.select { |element| element.matches?("author", ATOM) }
+    elements.empty? ? @feed_authors : elements
+  end
+end

data/lib/simple-rss/http_client.rb

@@ -0,0 +1,216 @@
+# rbs_inline: enabled
+
+require "net/http"
+require "timeout"
+require "openssl"
+require_relative "request_errors"
+require_relative "request_policy"
+
+class SimpleRSS::HTTPClient
+  ACCEPT = "application/feed+json, application/rss+xml, application/atom+xml, application/json, application/xml, text/xml, */*".freeze
+  REDIRECT_CODES = %w[301 302 303 307 308].freeze
+  CROSS_ORIGIN_HEADERS = %w[accept accept-language user-agent].freeze
+  PROTECTED_HEADERS = %w[host connection proxy-authorization proxy-connection accept-encoding range transfer-encoding content-length te trailer upgrade
+                         expect].freeze
+
+  # @rbs @options: Hash[Symbol, untyped]
+  # @rbs @headers: Hash[untyped, untyped]
+  # @rbs @policy: SimpleRSS::RequestPolicy?
+  # @rbs @remaining_bytes: Integer
+  # @rbs @remaining_wire_bytes: Integer
+  # @rbs @redirect_limit: Integer
+  # @rbs @timeout: untyped
+
+  # @rbs (Hash[Symbol, untyped]) -> void
+  def initialize(options)
+    @options = options.dup
+    @headers = (@options[:headers] || {}).dup
+    @policy = @options.key?(:network_policy) ? SimpleRSS::RequestPolicy.new(@options[:network_policy]) : nil
+    @remaining_bytes = @options.fetch(:max_bytes, 2 * 1024 * 1024)
+    @remaining_wire_bytes = @remaining_bytes
+    @redirect_limit = @options.fetch(:max_redirects, 5)
+    @timeout = @options.fetch(:timeout, @policy ? 10 : nil)
+    validate_options
+  end
+
+  # @rbs (String) -> [untyped, untyped]
+  def get(url)
+    return perform(URI.parse(url)) unless @policy
+
+    Timeout.timeout(@timeout, SimpleRSS::RequestTimeout, "HTTP operation exceeded its timeout") do
+      perform(SimpleRSS::RequestPolicy.parse_url(url))
+    end
+  rescue Timeout::Error
+    raise unless @policy
+
+    raise SimpleRSS::RequestTimeout, "HTTP operation exceeded its timeout"
+  rescue IOError, SystemCallError, SocketError, OpenSSL::SSL::SSLError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Resolv::ResolvError, Zlib::Error => e
+    raise unless @policy
+
+    raise SimpleRSS::RequestError, "HTTP transport failed: #{e.message}"
+  end
+
+  private
+
+  # @rbs () -> void
+  def validate_options
+    if !@policy && (@options.key?(:max_bytes) || @options.key?(:max_redirects))
+      raise ArgumentError, "max_bytes and max_redirects require network_policy"
+    end
+    return unless @policy
+
+    unless @timeout.is_a?(Numeric) && @timeout.real? && @timeout.finite? && @timeout.positive?
+      raise ArgumentError, "timeout must be a positive finite number"
+    end
+    unless @remaining_bytes.is_a?(Integer) && @remaining_bytes.positive?
+      raise ArgumentError, "max_bytes must be a positive integer"
+    end
+    unless @redirect_limit.is_a?(Integer) && @redirect_limit >= 0
+      raise ArgumentError, "max_redirects must be a nonnegative integer"
+    end
+    raise ArgumentError, "headers must be a hash" unless @headers.is_a?(Hash)
+
+    @headers.each do |name, value|
+      if PROTECTED_HEADERS.include?(name.to_s.downcase)
+        raise ArgumentError, "The transport controls the #{name} header"
+      end
+      unless name.to_s.match?(/\A[!#$%&'*+.^_`|~0-9A-Za-z-]+\z/) && value.is_a?(String) && !value.match?(/[\r\n\x00]/)
+        raise ArgumentError, "Invalid request header"
+      end
+    end
+  end
+
+  # @rbs (untyped) -> [untyped, untyped]
+  def perform(uri)
+    visited = {} #: Hash[String, bool]
+    redirects = @policy ? 0 : (@options[:_redirects] || 0)
+    redirect_error = @policy ? SimpleRSS::RedirectError : SimpleRSSError
+    loop do
+      raise SimpleRSS::RedirectError, "Redirect loop detected" if @policy && visited[uri.to_s]
+
+      visited[uri.to_s] = true
+      response = request(uri)
+      return [response, uri] unless redirect?(response)
+
+      location = response["Location"]
+      return [response, uri] unless location
+
+      redirects += 1
+      raise redirect_error, "Too many redirects" if redirects > @redirect_limit
+
+      next_uri = URI.join(uri.to_s, location)
+      next_uri = SimpleRSS::RequestPolicy.parse_url(next_uri.to_s) if @policy
+      strip_credentials if @policy && origin(uri) != origin(next_uri)
+      uri = next_uri
+    end
+  rescue URI::Error
+    raise unless @policy
+
+    raise SimpleRSS::PolicyError, "Malformed redirect URL"
+  end
+
+  # @rbs (untyped) -> bool
+  def redirect?(response)
+    return false if @options[:follow_redirects] == false
+    return REDIRECT_CODES.include?(response.code) if @policy
+
+    response.is_a?(Net::HTTPRedirection)
+  end
+
+  # @rbs (untyped) -> untyped
+  def request(uri)
+    http = build_http(uri)
+    request = Net::HTTP::Get.new(uri)
+    request["Accept"] = ACCEPT
+    request["User-Agent"] = "SimpleRSS/#{SimpleRSS::VERSION}"
+    request["If-None-Match"] = @options[:etag] if @options[:etag]
+    request["If-Modified-Since"] = @options[:last_modified] if @options[:last_modified]
+    @headers.each { |name, value| request[name] = value }
+    return http.request(request) unless @policy
+
+    request["Accept-Encoding"] = "gzip, deflate, identity"
+    http.request(request) { |response| read_response(response) }
+  end
+
+  # @rbs (untyped) -> untyped
+  def build_http(uri)
+    host = uri.hostname || raise(SimpleRSSError, "Invalid URL: missing host")
+    policy = @policy
+    http = policy ? Net::HTTP.new(host, uri.port, nil) : Net::HTTP.new(host, uri.port)
+    http.use_ssl = uri.scheme == "https"
+    if @timeout
+      http.open_timeout = @timeout
+      http.read_timeout = @timeout
+    end
+    return http unless policy
+
+    http.ipaddr = policy.address(uri)
+    http.write_timeout = @timeout
+    http.max_retries = 0
+    http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+    http.verify_hostname = true
+    http
+  end
+
+  # @rbs (untyped) -> void
+  def read_response(response)
+    return unless response.class.body_permitted?
+
+    encoding = response["Content-Encoding"].to_s.downcase
+    unless ["", "identity", "none", "gzip", "x-gzip", "deflate"].include?(encoding)
+      raise SimpleRSS::RequestError, "Unsupported Content-Encoding"
+    end
+    raise SimpleRSS::RequestError, "Partial HTTP responses are not supported" if response["Content-Range"] || response.code == "206"
+
+    body = +"".b
+    wire_bytes = 0
+    inflater = Zlib::Inflate.new(32 + Zlib::MAX_WBITS) if %w[gzip x-gzip deflate].include?(encoding)
+    response.read_body do |chunk|
+      if chunk.bytesize > @remaining_wire_bytes
+        raise SimpleRSS::ResponseTooLarge, "HTTP response bodies exceeded max_bytes"
+      end
+
+      @remaining_wire_bytes -= chunk.bytesize
+      wire_bytes += chunk.bytesize
+      unless inflater
+        append_body(body, chunk)
+        next
+      end
+      inflater.inflate(chunk) { |decoded| append_body(body, decoded) }
+      raise SimpleRSS::RequestError, "Trailing data in compressed HTTP response" if inflater.total_in != wire_bytes
+    end
+    raise SimpleRSS::RequestError, "Incomplete compressed HTTP response" if inflater && !inflater.finished?
+
+    declared_length = response.content_length
+    if declared_length && !response.chunked? && wire_bytes != declared_length
+      raise SimpleRSS::RequestError, "Incomplete HTTP response body"
+    end
+
+    response.delete("Content-Encoding")
+    response["Content-Length"] = body.bytesize.to_s if declared_length
+    response.body = body
+  ensure
+    inflater&.close
+  end
+
+  # @rbs (String, String) -> nil
+  def append_body(body, chunk)
+    raise SimpleRSS::ResponseTooLarge, "HTTP response bodies exceeded max_bytes" if chunk.bytesize > @remaining_bytes
+
+    @remaining_bytes -= chunk.bytesize
+    body << chunk
+    nil
+  end
+
+  # @rbs (untyped) -> Array[untyped]
+  def origin(uri)
+    [uri.scheme.downcase, uri.hostname.downcase, uri.port]
+  end
+
+  # @rbs () -> void
+  def strip_credentials
+    @headers = @headers.select { |name, _value| CROSS_ORIGIN_HEADERS.include?(name.to_s.downcase) }
+    @options.delete(:etag)
+    @options.delete(:last_modified)
+  end
+end