feedjira 0.9.0

data/benchmarks/feed_xml/xkcd.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en"><title>xkcd.com</title><link href="http://xkcd.com/" rel="alternate"></link><id>http://xkcd.com/</id><updated>2013-11-29T00:00:00Z</updated><entry><title>Oort Cloud</title><link href="http://xkcd.com/1297/" rel="alternate"></link><updated>2013-11-29T00:00:00Z</updated><id>http://xkcd.com/1297/</id><summary type="html">&lt;img src="http://imgs.xkcd.com/comics/oort_cloud.png" title="... I wanna try.  Hang on, be right back." alt="... I wanna try.  Hang on, be right back." /&gt;</summary></entry><entry><title>Git Commit</title><link href="http://xkcd.com/1296/" rel="alternate"></link><updated>2013-11-27T00:00:00Z</updated><id>http://xkcd.com/1296/</id><summary type="html">&lt;img src="http://imgs.xkcd.com/comics/git_commit.png" title="Merge branch 'asdfasjkfdlas/alkdjf' into sdkjfls-final" alt="Merge branch 'asdfasjkfdlas/alkdjf' into sdkjfls-final" /&gt;</summary></entry><entry><title>New Study</title><link href="http://xkcd.com/1295/" rel="alternate"></link><updated>2013-11-25T00:00:00Z</updated><id>http://xkcd.com/1295/</id><summary type="html">&lt;img src="http://imgs.xkcd.com/comics/new_study.png" title="When the results are published, no one will be sure whether to report on them again." alt="When the results are published, no one will be sure whether to report on them again." /&gt;</summary></entry><entry><title>Telescope Names</title><link href="http://xkcd.com/1294/" rel="alternate"></link><updated>2013-11-22T00:00:00Z</updated><id>http://xkcd.com/1294/</id><summary type="html">&lt;img src="http://imgs.xkcd.com/comics/telescope_names.png" title="The Thirty Meter Telescope will be renamed The Flesh-Searing Eye on the Volcano." alt="The Thirty Meter Telescope will be renamed The Flesh-Searing Eye on the Volcano." /&gt;</summary></entry></feed>

data/benchmarks/fetching_systems.rb

@@ -0,0 +1,23 @@
+require 'benchmark'
+require 'net/http'
+require 'curb'
+
+urls = ['http://www.google.com'] * 100
+
+Benchmark.bm(11) do |b|
+  b.report('Net::HTTP') do
+    urls.each do |url|
+      Net::HTTP.get URI.parse url
+    end
+  end
+
+  b.report('Curl::Easy') do
+    urls.each do |url|
+      Curl::Easy.perform url
+    end
+  end
+
+  b.report('Curl::Multi') do
+    Curl::Multi.get urls
+  end
+end

data/benchmarks/other_libraries.rb

@@ -0,0 +1,73 @@
+require 'benchmark'
+require 'feedjira'
+require 'simple-rss'
+require 'feed-normalizer'
+require 'feed_me'
+
+iterations = 10
+urls = File.readlines(File.dirname(__FILE__) + '/feed_list.txt')
+files = Dir.glob(File.dirname(__FILE__) + '/feed_xml/*.xml')
+xmls = files.map { |file| File.open(file).read }
+
+# suppress warnings
+$VERBOSE = nil
+
+puts 'Parsing benchmarks'
+
+Benchmark.bm(15) do |b|
+  b.report('feedjira') do
+    iterations.times do
+      xmls.each { |xml| Feedjira::Feed.parse xml }
+    end
+  end
+
+  b.report('simple-rss') do
+    iterations.times do
+      xmls.each { |xml| SimpleRSS.parse xml }
+    end
+  end
+
+  b.report('feed-normalizer') do
+    iterations.times do
+      xmls.each { |xml| FeedNormalizer::FeedNormalizer.parse xml }
+    end
+  end
+
+  # incompatible with `ruby-feedparser`, same constant used
+  require 'feed_parser'
+  b.report('feed_parser') do
+    iterations.times do
+      xmls.each { |xml| FeedParser.new(feed_xml: xml).parse }
+    end
+  end
+
+  b.report('feed_me') do
+    iterations.times do
+      xmls.each { |xml| FeedMe.parse xml }
+    end
+  end
+
+  # incompatible with `feed_parser`, same constant used
+  # require 'feedparser'
+  # b.report('ruby-feedparser') do
+  #   iterations.times do
+  #     xmls.each { |xml| FeedParser::Feed::new xml }
+  #   end
+  # end
+end
+
+puts "\nFetch and parse benchmarks"
+
+Benchmark.bm(15) do |b|
+  b.report('feedjira') do
+    iterations.times { Feedjira::Feed.fetch_and_parse urls }
+  end
+
+  # incompatible with `ruby-feedparser`, same constant used
+  require 'feed_parser'
+  b.report('feed_parser') do
+    iterations.times do
+      urls.each { |url| FeedParser.new(url: url).parse }
+    end
+  end
+end

data/feedjira.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/feedjira/version', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name    = 'feedjira'
+  s.version = Feedjira::VERSION
+  s.license = 'MIT'
+
+  s.authors  = ['Paul Dix', 'Julien Kirch', 'Ezekiel Templin', 'Jon Allured']
+  s.email    = 'feedjira@gmail.com'
+  s.homepage = 'http://feedjira.com'
+
+  s.summary     = 'A feed fetching and parsing library'
+  s.description = 'A library designed to retrieve and parse feeds as quickly as possible'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths = ['lib']
+
+  s.platform = Gem::Platform::RUBY
+
+  s.add_dependency 'sax-machine', '~> 0.2.1'
+  s.add_dependency 'curb',        '~> 0.8.1'
+  s.add_dependency 'loofah',      '~> 1.2.1'
+
+  s.add_development_dependency 'rspec', '~> 2.14.0'
+end

data/lib/feedjira.rb

@@ -0,0 +1,16 @@
+require 'zlib'
+require 'curb'
+require 'sax-machine'
+require 'loofah'
+
+require 'feedjira/core_ext'
+require 'feedjira/version'
+
+module Feedjira
+  autoload :FeedEntryUtilities, 'feedjira/feed_entry_utilities'
+  autoload :FeedUtilities,      'feedjira/feed_utilities'
+  autoload :Feed,               'feedjira/feed'
+  autoload :Parser,             'feedjira/parser'
+
+  class NoParserAvailable < StandardError; end
+end

data/lib/feedjira/core_ext.rb

@@ -0,0 +1,3 @@
+require "feedjira/core_ext/time"
+require "feedjira/core_ext/date"
+require "feedjira/core_ext/string"

data/lib/feedjira/core_ext/date.rb

@@ -0,0 +1,19 @@
+# Date code pulled and adapted from:
+# Ruby Cookbook by Lucas Carlson and Leonard Richardson
+# Published by O'Reilly
+# ISBN: 0-596-52369-6
+class Date
+  def feed_utils_to_gm_time
+    feed_utils_to_time(new_offset, :gm)
+  end
+
+  def feed_utils_to_local_time
+    feed_utils_to_time(new_offset(DateTime.now.offset-offset), :local)
+  end
+
+  private
+  def feed_utils_to_time(dest, method)
+    Time.send(method, dest.year, dest.month, dest.day, dest.hour, dest.min,
+              dest.sec, dest.zone)
+  end
+end

data/lib/feedjira/core_ext/string.rb

@@ -0,0 +1,9 @@
+class String
+  def sanitize!
+    self.replace(sanitize)
+  end
+
+  def sanitize
+    Loofah.scrub_fragment(self, :prune).to_s
+  end
+end

data/lib/feedjira/core_ext/time.rb

@@ -0,0 +1,31 @@
+require "time"
+require "date"
+
+class Time
+  # Parse a time string and convert it to UTC without raising errors.
+  # Parses a flattened 14-digit time (YYYYmmddHHMMMSS) as UTC.
+  #
+  # === Parameters
+  # [dt<String or Time>] Time definition to be parsed.
+  #
+  # === Returns
+  # A Time instance in UTC or nil if there were errors while parsing.
+  def self.parse_safely(dt)
+    if dt
+      case
+      when dt.is_a?(Time)
+        dt.utc
+      when dt.respond_to?(:empty?) && dt.empty?
+        nil
+      when dt.respond_to?(:to_datetime)
+        dt.to_datetime.utc
+      when dt.to_s =~ /\A\d{14}\z/
+        parse("#{dt.to_s}Z", true)
+      else
+        parse(dt.to_s, true).utc
+      end
+    end
+  rescue StandardError
+    nil
+  end unless method_defined?(:parse_safely)
+end

data/lib/feedjira/feed.rb

@@ -0,0 +1,459 @@
+module Feedjira
+  class Feed
+    USER_AGENT = 'feedjira http://feedjira.com'
+
+    # Passes raw XML and callbacks to a parser.
+    # === Parameters
+    # [parser<Object>] The parser to pass arguments to - must respond to
+    # `parse` and should return a Feed object.
+    # [xml<String>] The XML that you would like parsed.
+    # === Returns
+    # An instance of the parser feed type.
+    def self.parse_with(parser, xml, &block)
+      parser.parse xml, &block
+    end
+
+    # Takes a raw XML feed and attempts to parse it. If no parser is available a Feedjira::NoParserAvailable exception is raised.
+    # You can pass a block to be called when there's an error during the parsing.
+    # === Parameters
+    # [xml<String>] The XML that you would like parsed.
+    # === Returns
+    # An instance of the determined feed type. By default, one of these:
+    # * Feedjira::Parser::RSSFeedBurner
+    # * Feedjira::Parser::GoogleDocsAtom
+    # * Feedjira::Parser::AtomFeedBurner
+    # * Feedjira::Parser::Atom
+    # * Feedjira::Parser::ITunesRSS
+    # * Feedjira::Parser::RSS
+    # === Raises
+    # Feedjira::NoParserAvailable : If no valid parser classes could be found for the feed.
+    def self.parse(xml, &block)
+      if parser = determine_feed_parser_for_xml(xml)
+        parse_with parser, xml, &block
+      else
+        raise NoParserAvailable.new("No valid parser for XML.")
+      end
+    end
+
+    # Determines the correct parser class to use for parsing the feed.
+    #
+    # === Parameters
+    # [xml<String>] The XML that you would like determine the parser for.
+    # === Returns
+    # The class name of the parser that can handle the XML.
+    def self.determine_feed_parser_for_xml(xml)
+      start_of_doc = xml.slice(0, 2000)
+      feed_classes.detect {|klass| klass.able_to_parse?(start_of_doc)}
+    end
+
+    # Adds a new feed parsing class that will be used for parsing.
+    #
+    # === Parameters
+    # [klass<Constant>] The class/constant that you want to register.
+    # === Returns
+    # A updated array of feed parser class names.
+    def self.add_feed_class(klass)
+      feed_classes.unshift klass
+    end
+
+    # Provides a list of registered feed parsing classes.
+    #
+    # === Returns
+    # A array of class names.
+    def self.feed_classes
+      @feed_classes ||= [
+        Feedjira::Parser::RSSFeedBurner,
+        Feedjira::Parser::GoogleDocsAtom,
+        Feedjira::Parser::AtomFeedBurner,
+        Feedjira::Parser::Atom,
+        Feedjira::Parser::ITunesRSS,
+        Feedjira::Parser::RSS
+      ]
+    end
+
+    # Makes all registered feeds types look for the passed in element to parse.
+    # This is actually just a call to element (a SAXMachine call) in the class.
+    #
+    # === Parameters
+    # [element_tag<String>] The element tag
+    # [options<Hash>] Valid keys are same as with SAXMachine
+    def self.add_common_feed_element(element_tag, options = {})
+      feed_classes.each do |k|
+        k.element element_tag, options
+      end
+    end
+
+    # Makes all registered feeds types look for the passed in elements to parse.
+    # This is actually just a call to elements (a SAXMachine call) in the class.
+    #
+    # === Parameters
+    # [element_tag<String>] The element tag
+    # [options<Hash>] Valid keys are same as with SAXMachine
+    def self.add_common_feed_elements(element_tag, options = {})
+      feed_classes.each do |k|
+        k.elements element_tag, options
+      end
+    end
+
+    # Makes all registered entry types look for the passed in element to parse.
+    # This is actually just a call to element (a SAXMachine call) in the class.
+    #
+    # === Parameters
+    # [element_tag<String>]
+    # [options<Hash>] Valid keys are same as with SAXMachine
+    def self.add_common_feed_entry_element(element_tag, options = {})
+      call_on_each_feed_entry :element, element_tag, options
+    end
+
+    # Makes all registered entry types look for the passed in elements to parse.
+    # This is actually just a call to element (a SAXMachine call) in the class.
+    #
+    # === Parameters
+    # [element_tag<String>]
+    # [options<Hash>] Valid keys are same as with SAXMachine
+    def self.add_common_feed_entry_elements(element_tag, options = {})
+      call_on_each_feed_entry :elements, element_tag, options
+    end
+
+    # Call a method on all feed entries classes.
+    #
+    # === Parameters
+    # [method<Symbol>] The method name
+    # [parameters<Array>] The method parameters
+    def self.call_on_each_feed_entry(method, *parameters)
+      feed_classes.each do |k|
+        # iterate on the collections defined in the sax collection
+        k.sax_config.collection_elements.each_value do |vl|
+          # vl is a list of CollectionConfig mapped to an attribute name
+          # we'll look for the one set as 'entries' and add the new element
+          vl.find_all{|v| (v.accessor == 'entries') && (v.data_class.class == Class)}.each do |v|
+              v.data_class.send(method, *parameters)
+          end
+        end
+      end
+    end
+
+    # Setup curl from options.
+    # Possible parameters:
+    # * :user_agent          - overrides the default user agent.
+    # * :compress            - any value to enable compression
+    # * :enable_cookies      - boolean
+    # * :cookiefile          - file to read cookies
+    # * :cookies             - contents of cookies header
+    # * :http_authentication - array containing username, then password
+    # * :proxy_url           - proxy url
+    # * :proxy_port          - proxy port
+    # * :max_redirects       - max number of redirections
+    # * :timeout             - timeout
+    # * :ssl_verify_host     - boolean
+    # * :ssl_verify_peer     - boolean
+    # * :ssl_version         - the ssl version to use, see OpenSSL::SSL::SSLContext::METHODS for options
+    def self.setup_easy(curl, options={})
+      curl.headers["Accept-encoding"]   = 'gzip, deflate' if options.has_key?(:compress)
+      curl.headers["User-Agent"]        = (options[:user_agent] || USER_AGENT)
+      curl.enable_cookies               = options[:enable_cookies] if options.has_key?(:enable_cookies)
+      curl.cookiefile                   = options[:cookiefile] if options.has_key?(:cookiefile)
+      curl.cookies                      = options[:cookies] if options.has_key?(:cookies)
+
+      curl.userpwd = options[:http_authentication].join(':') if options.has_key?(:http_authentication)
+      curl.proxy_url = options[:proxy_url] if options.has_key?(:proxy_url)
+      curl.proxy_port = options[:proxy_port] if options.has_key?(:proxy_port)
+      curl.max_redirects = options[:max_redirects] if options[:max_redirects]
+      curl.timeout = options[:timeout] if options[:timeout]
+      curl.ssl_verify_host = options[:ssl_verify_host] if options.has_key?(:ssl_verify_host)
+      curl.ssl_verify_peer = options[:ssl_verify_peer] if options.has_key?(:ssl_verify_peer)
+      curl.ssl_version = options[:ssl_version] if options.has_key?(:ssl_version)
+
+      curl.follow_location = true
+    end
+
+    # Fetches and returns the raw XML for each URL provided.
+    #
+    # === Parameters
+    # [urls<String> or <Array>] A single feed URL, or an array of feed URLs.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 :if_modified_since - Time object representing when the feed was last updated.
+    #                 :if_none_match - String that's normally an etag for the request that was stored previously.
+    #                 :on_success - Block that gets executed after a successful request.
+    #                 :on_failure - Block that gets executed after a failed request.
+    #                 * all parameters defined in setup_easy
+    # === Returns
+    # A String of XML if a single URL is passed.
+    #
+    # A Hash if multiple URL's are passed. The key will be the URL, and the value the XML.
+    def self.fetch_raw(urls, options = {})
+      url_queue = [*urls]
+      multi = Curl::Multi.new
+      responses = {}
+      url_queue.each do |url|
+        easy = Curl::Easy.new(url) do |curl|
+          setup_easy curl, options
+
+          curl.headers["If-Modified-Since"] = options[:if_modified_since].httpdate if options.has_key?(:if_modified_since)
+          curl.headers["If-None-Match"]     = options[:if_none_match] if options.has_key?(:if_none_match)
+
+          curl.on_success do |c|
+            responses[url] = decode_content(c)
+          end
+
+          curl.on_complete do |c, err|
+            responses[url] = c.response_code unless responses.has_key?(url)
+          end
+        end
+        multi.add(easy)
+      end
+
+      multi.perform
+      urls.is_a?(String) ? responses.values.first : responses
+    end
+
+    # Fetches and returns the parsed XML for each URL provided.
+    #
+    # === Parameters
+    # [urls<String> or <Array>] A single feed URL, or an array of feed URLs.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    # * :user_agent - String that overrides the default user agent.
+    # * :if_modified_since - Time object representing when the feed was last updated.
+    # * :if_none_match - String, an etag for the request that was stored previously.
+    # * :on_success - Block that gets executed after a successful request.
+    # * :on_failure - Block that gets executed after a failed request.
+    # === Returns
+    # A Feed object if a single URL is passed.
+    #
+    # A Hash if multiple URL's are passed. The key will be the URL, and the value the Feed object.
+    def self.fetch_and_parse(urls, options = {})
+      url_queue = [*urls]
+      multi = Curl::Multi.new
+      responses = {}
+
+      # I broke these down so I would only try to do 30 simultaneously because
+      # I was getting weird errors when doing a lot. As one finishes it pops another off the queue.
+      url_queue.slice!(0, 30).each do |url|
+        add_url_to_multi(multi, url, url_queue, responses, options)
+      end
+
+      multi.perform
+      return urls.is_a?(String) ? responses.values.first : responses
+    end
+
+    # Decodes the XML document if it was compressed.
+    #
+    # === Parameters
+    # [curl_request<Curl::Easy>] The Curl::Easy response object from the request.
+    # === Returns
+    # A decoded string of XML.
+    def self.decode_content(c)
+      if c.header_str.match(/Content-Encoding: gzip/i)
+        begin
+          gz =  Zlib::GzipReader.new(StringIO.new(c.body_str))
+          xml = gz.read
+          gz.close
+        rescue Zlib::GzipFile::Error
+          # Maybe this is not gzipped?
+          xml = c.body_str
+        end
+      elsif c.header_str.match(/Content-Encoding: deflate/i)
+        xml = Zlib::Inflate.inflate(c.body_str)
+      else
+        xml = c.body_str
+      end
+
+      xml
+    end
+
+    # Updates each feed for each Feed object provided.
+    #
+    # === Parameters
+    # [feeds<Feed> or <Array>] A single feed object, or an array of feed objects.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 * :on_success - Block that gets executed after a successful request.
+    #                 * :on_failure - Block that gets executed after a failed request.
+    #                 * all parameters defined in setup_easy
+    # === Returns
+    # A updated Feed object if a single URL is passed.
+    #
+    # A Hash if multiple Feeds are passed. The key will be the URL, and the value the updated Feed object.
+    def self.update(feeds, options = {})
+      feed_queue = [*feeds]
+      multi = Curl::Multi.new
+      responses = {}
+
+      feed_queue.slice!(0, 30).each do |feed|
+        add_feed_to_multi(multi, feed, feed_queue, responses, options)
+      end
+
+      multi.perform
+      feeds.is_a?(Array) ? responses : responses.values.first
+    end
+
+    # An abstraction for adding a feed by URL to the passed Curb::multi stack.
+    #
+    # === Parameters
+    # [multi<Curl::Multi>] The Curl::Multi object that the request should be added too.
+    # [url<String>] The URL of the feed that you would like to be fetched.
+    # [url_queue<Array>] An array of URLs that are queued for request.
+    # [responses<Hash>] Existing responses that you want the response from the request added to.
+    # [feeds<String> or <Array>] A single feed object, or an array of feed objects.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 * :on_success - Block that gets executed after a successful request.
+    #                 * :on_failure - Block that gets executed after a failed request.
+    #                 * all parameters defined in setup_easy
+    # === Returns
+    # The updated Curl::Multi object with the request details added to it's stack.
+    def self.add_url_to_multi(multi, url, url_queue, responses, options)
+      easy = Curl::Easy.new(url) do |curl|
+        setup_easy curl, options
+        curl.headers["If-Modified-Since"] = options[:if_modified_since].httpdate if options.has_key?(:if_modified_since)
+        curl.headers["If-None-Match"]     = options[:if_none_match] if options.has_key?(:if_none_match)
+
+        curl.on_success do |c|
+          xml = decode_content(c)
+          klass = determine_feed_parser_for_xml(xml)
+
+          if klass
+            begin
+              feed = parse_with klass, xml, &on_parser_failure(url)
+
+              feed.feed_url = c.last_effective_url
+              feed.etag = etag_from_header(c.header_str)
+              feed.last_modified = last_modified_from_header(c.header_str)
+              responses[url] = feed
+              options[:on_success].call(url, feed) if options.has_key?(:on_success)
+            rescue Exception => e
+              call_on_failure(c, e, options[:on_failure])
+            end
+          else
+            call_on_failure(c, "Can't determine a parser", options[:on_failure])
+          end
+        end
+
+        #
+        # trigger on_failure for 404s
+        #
+        curl.on_complete do |c|
+          add_url_to_multi(multi, url_queue.shift, url_queue, responses, options) unless url_queue.empty?
+          responses[url] = c.response_code unless responses.has_key?(url)
+        end
+
+        curl.on_redirect do |c|
+          if c.response_code == 304 # it's not modified. this isn't an error condition
+            options[:on_success].call(url, nil) if options.has_key?(:on_success)
+          end
+        end
+
+        curl.on_missing do |c|
+          if c.response_code == 404 && options.has_key?(:on_failure)
+            call_on_failure(c, 'Server returned a 404', options[:on_failure])
+          end
+        end
+
+        curl.on_failure do |c, err|
+          responses[url] = c.response_code
+          call_on_failure(c, err, options[:on_failure])
+        end
+      end
+      multi.add(easy)
+    end
+
+    # An abstraction for adding a feed by a Feed object to the passed Curb::multi stack.
+    #
+    # === Parameters
+    # [multi<Curl::Multi>] The Curl::Multi object that the request should be added too.
+    # [feed<Feed>] A feed object that you would like to be fetched.
+    # [url_queue<Array>] An array of feed objects that are queued for request.
+    # [responses<Hash>] Existing responses that you want the response from the request added to.
+    # [feeds<String>] or <Array> A single feed object, or an array of feed objects.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 * :on_success - Block that gets executed after a successful request.
+    #                 * :on_failure - Block that gets executed after a failed request.
+    #                 * all parameters defined in setup_easy
+    # === Returns
+    # The updated Curl::Multi object with the request details added to it's stack.
+    def self.add_feed_to_multi(multi, feed, feed_queue, responses, options)
+      easy = Curl::Easy.new(feed.feed_url) do |curl|
+        setup_easy curl, options
+        curl.headers["If-Modified-Since"] = feed.last_modified.httpdate if feed.last_modified
+        curl.headers["If-Modified-Since"] = options[:if_modified_since] if options[:if_modified_since] && (!feed.last_modified || (Time.parse(options[:if_modified_since].to_s) > feed.last_modified))
+        curl.headers["If-None-Match"]     = feed.etag if feed.etag
+
+        curl.on_success do |c|
+          begin
+            updated_feed = Feed.parse c.body_str, &on_parser_failure(feed.feed_url)
+
+            updated_feed.feed_url = c.last_effective_url
+            updated_feed.etag = etag_from_header(c.header_str)
+            updated_feed.last_modified = last_modified_from_header(c.header_str)
+            feed.update_from_feed(updated_feed)
+            responses[feed.feed_url] = feed
+            options[:on_success].call(feed) if options.has_key?(:on_success)
+          rescue Exception => e
+            call_on_failure(c, e, options[:on_failure])
+          end
+        end
+
+        curl.on_failure do |c, err| # response code 50X
+          responses[feed.feed_url] = c.response_code
+          call_on_failure(c, 'Server returned a 404', options[:on_failure])
+        end
+
+        curl.on_redirect do |c, err| # response code 30X
+          if c.response_code == 304
+            options[:on_success].call(feed) if options.has_key?(:on_success)
+          else
+            responses[feed.feed_url] = c.response_code
+            call_on_failure(c, err, options[:on_failure])
+          end
+        end
+
+        curl.on_complete do |c|
+          add_feed_to_multi(multi, feed_queue.shift, feed_queue, responses, options) unless feed_queue.empty?
+          responses[feed.feed_url] = feed unless responses.has_key?(feed.feed_url)
+        end
+      end
+      multi.add(easy)
+    end
+
+    # Determines the etag from the request headers.
+    #
+    # === Parameters
+    # [header<String>] Raw request header returned from the request
+    # === Returns
+    # A string of the etag or nil if it cannot be found in the headers.
+    def self.etag_from_header(header)
+      header =~ /.*ETag:\s(.*)\r/
+      $1
+    end
+
+    # Determines the last modified date from the request headers.
+    #
+    # === Parameters
+    # [header<String>] Raw request header returned from the request
+    # === Returns
+    # A Time object of the last modified date or nil if it cannot be found in the headers.
+    def self.last_modified_from_header(header)
+      header =~ /.*Last-Modified:\s(.*)\r/
+      Time.parse_safely($1) if $1
+    end
+
+    class << self
+      private
+
+      def on_parser_failure(url)
+        Proc.new { |message| raise "Error while parsing [#{url}] #{message}" }
+      end
+
+      def call_on_failure(c, error, on_failure)
+        if on_failure
+          if on_failure.arity == 4
+            warn 'on_failure proc with deprecated arity 4 should include a fifth parameter containing the error'
+            on_failure.call(c.url, c.response_code, c.header_str, c.body_str)
+          elsif on_failure.arity == 2
+            on_failure.call(c, error)
+          else
+            warn "on_failure proc with invalid parameters number #{on_failure.arity} instead of 2, ignoring it"
+          end
+        end
+      end
+    end
+  end
+end