codders-feedzirra 0.2.0.rc2

data/.gitignore

@@ -0,0 +1,12 @@
+.DS_Store
+.rvmrc
+TODO
+Gemfile.lock
+rdoc/
+doc/
+pkg/
+coverage/
+.rbx
+.bundle
+*.swp
+*.swo

data/.rspec

@@ -0,0 +1 @@
+--color

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+rvm:
+  - 1.8.7
+  - 1.9.2
+  - 1.9.3
+  - rbx-18mode
+  - rbx-19mode
+notifications:
+  irc: "irc.freenode.org#feedzirra"

data/Gemfile

@@ -0,0 +1,10 @@
+source :rubygems
+
+gemspec
+
+group :development, :test do
+  gem 'rake'
+  gem 'guard-rspec'
+  gem 'growl', :require => false
+  gem 'simplecov', :require => false, :platforms => :mri_19
+end

data/Guardfile

@@ -0,0 +1,6 @@
+guard 'rspec', :rvm => ['1.9.3@feedzirra', 'rbx-head@feedzirra'] do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end
+

data/HISTORY.md

@@ -0,0 +1,25 @@
+# HEAD
+* Remove gorillib dependency [[#113](https://github.com/pauldix/feedzirra/pull/113)].
+
+# 0.2.0.rc2
+* Bump sax-machine to `v0.2.0.rc1`, fixes encoding issues [[#76](https://github.com/pauldix/feedzirra/issues/76)].
+
+# 0.2.0.rc1
+* Remove ActiveSupport dependency
+  * No longer tethered to any version of Rails!
+* Update curb (v0.8.0) and rspec (v2.10.0)
+* Revert [3008ceb](https://github.com/pauldix/feedzirra/commit/3008ceb338df1f4c37a211d0aab8a6ad4f584dbc)
+* Add Travis-CI integration
+* General repository and gem maintenance
+
+# 0.1.3
+* ?
+
+# 0.1.2
+* ?
+
+# 0.1.1
+* make FeedEntries enumerable (patch by Daniel Gregoire)
+
+# 0.1.0
+* lower builder requirement to make it rails-3 friendly

data/README.md

@@ -0,0 +1,179 @@
+# Feedzirra [![Build Status](https://secure.travis-ci.org/pauldix/feedzirra.png)](http://travis-ci.org/pauldix/feedzirra)
+
+I'd like feedback on the api and any bugs encountered on feeds in the wild. I've set up a [google group here](http://groups.google.com/group/feedzirra).
+
+## Description
+
+Feedzirra is a feed library that is designed to get and update many feeds as quickly as possible. This includes using libcurl-multi through the [curb](https://github.com/taf2/curb) gem for faster http gets, and libxml through [nokogiri](https://github.com/tenderlove/nokogiri) and [sax-machine](https://github.com/pauldix/sax-machine) for faster parsing.
+
+Once you have fetched feeds using Feedzirra, they can be updated using the feed objects. Feedzirra automatically inserts etag and last-modified information from the http response headers to lower bandwidth usage, eliminate unnecessary parsing, and make things speedier in general.
+
+Another feature present in Feedzirra is the ability to create callback functions that get called "on success" and "on failure" when getting a feed. This makes it easy to do things like log errors or update data stores.
+
+The fetching and parsing logic have been decoupled so that either of them can be used in isolation if you'd prefer not to use everything that Feedzirra offers. However, the code examples below use helper methods in the Feed class that put everything together to make things as simple as possible.
+
+The final feature of Feedzirra is the ability to define custom parsing classes. In truth, Feedzirra could be used to parse much more than feeds. Microformats, page scraping, and almost anything else are fair game.
+
+## Speedup date parsing
+
+In MRI the date parsing code is written in ruby and is optimized for readability over speed, to speed up this part you can install the [home_run](https://github.com/jeremyevans/home_run) gem to replace it with an optimized C version.
+
+## Usage
+
+[A gist of the following code](http://gist.github.com/57285)
+
+    require 'feedzirra'
+
+    # fetching a single feed
+    feed = Feedzirra::Feed.fetch_and_parse("http://feeds.feedburner.com/PaulDixExplainsNothing")
+
+    # feed and entries accessors
+    feed.title          # => "Paul Dix Explains Nothing"
+    feed.url            # => "http://www.pauldix.net"
+    feed.feed_url       # => "http://feeds.feedburner.com/PaulDixExplainsNothing"
+    feed.etag           # => "GunxqnEP4NeYhrqq9TyVKTuDnh0"
+    feed.last_modified  # => Sat Jan 31 17:58:16 -0500 2009 # it's a Time object
+
+    entry = feed.entries.first
+    entry.title      # => "Ruby Http Client Library Performance"
+    entry.url        # => "http://www.pauldix.net/2009/01/ruby-http-client-library-performance.html"
+    entry.author     # => "Paul Dix"
+    entry.summary    # => "..."
+    entry.content    # => "..."
+    entry.published  # => Thu Jan 29 17:00:19 UTC 2009 # it's a Time object
+    entry.categories # => ["...", "..."]
+
+    # sanitizing an entry's content
+    entry.title.sanitize   # => returns the title with harmful stuff escaped
+    entry.author.sanitize  # => returns the author with harmful stuff escaped
+    entry.content.sanitize # => returns the content with harmful stuff escaped
+    entry.content.sanitize! # => returns content with harmful stuff escaped and replaces original (also exists for author and title)
+    entry.sanitize!         # => sanitizes the entry's title, author, and content in place (as in, it changes the value to clean versions)
+    feed.sanitize_entries!  # => sanitizes all entries in place
+
+    # updating a single feed
+    updated_feed = Feedzirra::Feed.update(feed)
+
+    # an updated feed has the following extra accessors
+    updated_feed.updated?     # returns true if any of the feed attributes have been modified. will return false if only new entries
+    updated_feed.new_entries  # a collection of the entry objects that are newer than the latest in the feed before update
+
+    # fetching multiple feeds
+    feed_urls = ["http://feeds.feedburner.com/PaulDixExplainsNothing", "http://feeds.feedburner.com/trottercashion"]
+    feeds = Feedzirra::Feed.fetch_and_parse(feed_urls)
+
+    # feeds is now a hash with the feed_urls as keys and the parsed feed objects as values. If an error was thrown
+    # there will be a Fixnum of the http response code instead of a feed object
+
+    # updating multiple feeds. it expects a collection of feed objects
+    updated_feeds = Feedzirra::Feed.update(feeds.values)
+
+    # defining custom behavior on failure or success. note that a return status of 304 (not updated) will call the on_success handler
+    feed = Feedzirra::Feed.fetch_and_parse("http://feeds.feedburner.com/PaulDixExplainsNothing",
+        :on_success => lambda [|url, feed| puts feed.title ],
+        :on_failure => lambda [|url, response_code, response_header, response_body| puts response_body ])
+    # if a collection was passed into fetch_and_parse, the handlers will be called for each one
+
+    # the behavior for the handlers when using Feedzirra::Feed.update is slightly different. The feed passed into on_success will be
+    # the updated feed with the standard updated accessors. on failure it will be the original feed object passed into update
+
+    # fetching a feed via a proxy (optional)
+    feed = Feedzirra::Feed.fetch_and_parse("http://feeds.feedburner.com/PaulDixExplainsNothing", [:proxy_url => '10.0.0.1', :proxy_port => 3084])
+
+## Extending
+
+### Adding a feed parsing class
+
+    # Adds a new feed parsing class, this class will be used first
+    Feedzirra::Feed.add_feed_class MyFeedClass
+
+### Adding attributes to all feeds types / all entries types
+
+    # Add the generator attribute to all feed types
+    Feedzirra::Feed.add_common_feed_element('generator')
+    Feedzirra::Feed.fetch_and_parse("href="http://www.pauldix.net/atom.xml").generator # => 'TypePad'
+
+    # Add some GeoRss information
+    Feedzirra::Feed.add_common_feed_entry_element('geo:lat', :as => :lat)
+    Feedzirra::Feed.fetch_and_parse("http://www.earthpublisher.com/georss.php").entries.each do |e|
+        p "lat: #[e.lat}, long: #{e.long]"
+    end
+
+### Adding attributes to only one class
+
+If you want to add attributes for only on class you simply have to declare them in the class
+
+    # Add some GeoRss information
+    require 'lib/feedzirra/parser/rss_entry'
+
+    class Feedzirra::Parser::RSSEntry
+        element 'geo:lat', :as => :lat
+        element 'geo:long', :as => :long
+    end
+
+    # Fetch a feed containing GeoRss info and print them
+    Feedzirra::Feed.fetch_and_parse("http://www.earthpublisher.com/georss.php").entries.each do |e|
+        p "lat: #[e.lat}, long: #{e.long]"
+    end
+
+## Benchmarks
+
+One of the goals of Feedzirra is speed. This includes not only parsing, but fetching multiple feeds as quickly as possible. I ran a benchmark getting 20 feeds 10 times using Feedzirra, rFeedParser, and FeedNormalizer. For more details the [benchmark code can be found in the project in spec/benchmarks/feedzirra_benchmarks.rb](https://github.com/pauldix/feedzirra/blob/7fb5634c5c16e9c6ec971767b462c6518cd55f5d/spec/benchmarks/feedzirra_benchmarks.rb)
+
+    feedzirra          5.170000   1.290000   6.460000 ( 18.917796)
+    rfeedparser      104.260000  12.220000 116.480000 (244.799063)
+    feed-normalizer   66.250000   4.010000  70.260000 (191.589862)
+
+The result of that benchmark is a bit sketchy because of the network variability. Running 10 times against the same 20 feeds was meant to smooth some of that out. However, there is also a [benchmark comparing parsing speed in spec/benchmarks/parsing_benchmark.rb](https://github.com/pauldix/feedzirra/blob/7fb5634c5c16e9c6ec971767b462c6518cd55f5d/spec/benchmarks/parsing_benchmark.rb) on an atom feed.
+
+    feedzirra        0.500000   0.030000   0.530000 (  0.658744)
+    rfeedparser      8.400000   1.110000   9.510000 ( 11.839827)
+    feed-normalizer  5.980000   0.160000   6.140000 (  7.576140)
+
+There's also a [benchmark that shows the results of using Feedzirra to perform updates on feeds](https://github.com/pauldix/feedzirra/blob/45d64319544c61a4c9eb9f7f825c73b9f9030cb3/spec/benchmarks/updating_benchmarks.rb) you've already pulled in. I tested against 179 feeds. The first is the initial pull and the second is an update 65 seconds later. I'm not sure how many of them support etag and last-modified, so performance may be better or worse depending on what feeds you're requesting.
+
+    feedzirra fetch and parse  4.010000   0.710000   4.720000 ( 15.110101)
+    feedzirra update           0.660000   0.280000   0.940000 (  5.152709)
+
+## TODO
+
+This thing needs to hammer on many different feeds in the wild. I'm sure there will be bugs. I want to find them and crush them. I didn't bother using the test suite for feedparser. i wanted to start fresh.
+
+Here are some more specific TODOs.
+* Make a feedzirra-rails gem to integrate feedzirra seamlessly with Rails and ActiveRecord.
+* Add support for authenticated feeds.
+* Create a super sweet DSL for defining new parsers.
+* Test against Ruby 1.9.1 and fix any bugs.
+* I'm not keeping track of modified on entries. Should I add this?
+* Clean up the fetching code inside feed.rb so it doesn't suck so hard.
+* Make the feed_spec actually mock stuff out so it doesn't hit the net.
+* Readdress how feeds determine if they can parse a document. Maybe I should use namespaces instead?
+
+## LICENSE
+
+(The MIT License)
+ 
+Copyright (c) 2009-2012:
+ 
+- [Paul Dix](http://pauldix.net)
+- [Julien Kirch](http://archiloque.net/)
+- [Ezekiel Templin](http://zeke.templ.in/)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+ 
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+ 
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,6 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+task :test => :spec
+task :default => :test

data/feedzirra.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/feedzirra/version', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name    = 'codders-feedzirra'
+  s.version = Feedzirra::VERSION
+
+  s.authors  = ['Paul Dix', 'Julien Kirch', "Ezekiel Templin"]
+  s.date     = Date.today
+  s.email    = 'feedzirra@googlegroups.com'
+  s.homepage = 'http://github.com/codders/feedzirra'
+
+  s.summary     = 'A feed fetching and parsing library'
+  s.description = 'A feed fetching and parsing library that treats the internet like Godzilla treats Japan: it dominates and eats all.'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths = ['lib']
+  
+  s.platform = Gem::Platform::RUBY
+
+  s.add_dependency 'nokogiri',          '~> 1.5.3'
+  s.add_dependency 'sax-machine',       '~> 0.2.0.rc1'
+  s.add_dependency 'codders-curb',              '~> 0.8.0'
+  s.add_dependency 'loofah',            '~> 1.2.1'
+
+  s.add_development_dependency 'rspec', '~> 2.10.0'
+end

data/lib/feedzirra.rb

@@ -0,0 +1,17 @@
+require 'zlib'
+require 'curb'
+require 'sax-machine'
+require 'loofah'
+require 'uri'
+
+require 'feedzirra/core_ext'
+require 'feedzirra/version'
+
+module Feedzirra
+  autoload :FeedEntryUtilities, 'feedzirra/feed_entry_utilities'
+  autoload :FeedUtilities,      'feedzirra/feed_utilities'
+  autoload :Feed,               'feedzirra/feed'
+  autoload :Parser,             'feedzirra/parser'
+  
+  class NoParserAvailable < StandardError; end
+end

data/lib/feedzirra/core_ext.rb

@@ -0,0 +1,3 @@
+require "feedzirra/core_ext/time"
+require "feedzirra/core_ext/date"
+require "feedzirra/core_ext/string"

data/lib/feedzirra/core_ext/date.rb

@@ -0,0 +1,19 @@
+# Date code pulled and adapted from:
+# Ruby Cookbook by Lucas Carlson and Leonard Richardson
+# Published by O'Reilly
+# ISBN: 0-596-52369-6
+class Date
+  def feed_utils_to_gm_time
+    feed_utils_to_time(new_offset, :gm)
+  end
+
+  def feed_utils_to_local_time
+    feed_utils_to_time(new_offset(DateTime.now.offset-offset), :local)
+  end
+
+  private
+  def feed_utils_to_time(dest, method)
+    Time.send(method, dest.year, dest.month, dest.day, dest.hour, dest.min,
+              dest.sec, dest.zone)
+  end
+end

data/lib/feedzirra/core_ext/string.rb

@@ -0,0 +1,9 @@
+class String
+  def sanitize!
+    self.replace(sanitize)
+  end
+  
+  def sanitize
+    Loofah.scrub_fragment(self, :prune).to_s
+  end
+end

data/lib/feedzirra/core_ext/time.rb

@@ -0,0 +1,29 @@
+require "time"
+require "date"
+
+class Time
+  # Parse a time string and convert it to UTC without raising errors.
+  # Parses a flattened 14-digit time (YYYYmmddHHMMMSS) as UTC.
+  #
+  # === Parameters
+  # [dt<String or Time>] Time definition to be parsed.
+  #
+  # === Returns
+  # A Time instance in UTC or nil if there were errors while parsing.
+  def self.parse_safely(dt)
+    if dt
+      case
+      when dt.is_a?(Time)
+        dt.utc
+      when dt.respond_to?(:empty?) && dt.empty?
+        nil
+      when dt.to_s =~ /\A\d{14}\z/
+        parse("#{dt.to_s}Z", true)
+      else
+        parse(dt.to_s, true).utc
+      end
+    end
+  rescue StandardError
+    nil
+  end unless method_defined?(:parse_safely)
+end

data/lib/feedzirra/feed.rb

@@ -0,0 +1,382 @@
+module Feedzirra  
+  class Feed
+    USER_AGENT = "feedzirra http://github.com/pauldix/feedzirra/tree/master"
+    
+    # Takes a raw XML feed and attempts to parse it. If no parser is available a Feedzirra::NoParserAvailable exception is raised.
+    # You can pass a block to be called when there's an error during the parsing.
+    # === Parameters
+    # [xml<String>] The XML that you would like parsed.
+    # === Returns
+    # An instance of the determined feed type. By default a Feedzirra::Atom, Feedzirra::AtomFeedBurner, Feedzirra::RDF, or Feedzirra::RSS object.
+    # === Raises
+    # Feedzirra::NoParserAvailable : If no valid parser classes could be found for the feed.
+    def self.parse(xml, &block)
+      if parser = determine_feed_parser_for_xml(xml)
+        parser.parse(xml, block)
+      else
+        raise NoParserAvailable.new("No valid parser for XML.")
+      end
+    end
+
+    # Determines the correct parser class to use for parsing the feed.
+    # 
+    # === Parameters
+    # [xml<String>] The XML that you would like determine the parser for.
+    # === Returns
+    # The class name of the parser that can handle the XML.
+    def self.determine_feed_parser_for_xml(xml)
+      start_of_doc = xml.slice(0, 2000)
+      feed_classes.detect {|klass| klass.able_to_parse?(start_of_doc)}
+    end
+
+    # Adds a new feed parsing class that will be used for parsing.
+    #
+    # === Parameters
+    # [klass<Constant>] The class/constant that you want to register.
+    # === Returns
+    # A updated array of feed parser class names.
+    def self.add_feed_class(klass) 
+      feed_classes.unshift klass
+    end
+
+    # Provides a list of registered feed parsing classes.
+    #
+    # === Returns
+    # A array of class names.
+    def self.feed_classes
+      @feed_classes ||= [Feedzirra::Parser::RSSFeedBurner, Feedzirra::Parser::RSS, Feedzirra::Parser::GoogleDocsAtom, Feedzirra::Parser::AtomFeedBurner, Feedzirra::Parser::Atom, Feedzirra::Parser::ITunesRSS]
+    end
+    
+    # Makes all registered feeds types look for the passed in element to parse.
+    # This is actually just a call to element (a SAXMachine call) in the class.
+    #
+    # === Parameters
+    # [element_tag<String>] The element tag
+    # [options<Hash>] Valid keys are same as with SAXMachine
+    def self.add_common_feed_element(element_tag, options = {})
+      feed_classes.each do |k|
+        k.element element_tag, options
+      end
+    end
+
+    # Makes all registered feeds types look for the passed in elements to parse.
+    # This is actually just a call to elements (a SAXMachine call) in the class.
+    #
+    # === Parameters
+    # [element_tag<String>] The element tag
+    # [options<Hash>] Valid keys are same as with SAXMachine
+    def self.add_common_feed_elements(element_tag, options = {})
+      feed_classes.each do |k|
+        k.elements element_tag, options
+      end
+    end
+
+    # Makes all registered entry types look for the passed in element to parse.
+    # This is actually just a call to element (a SAXMachine call) in the class.
+    #
+    # === Parameters
+    # [element_tag<String>]
+    # [options<Hash>] Valid keys are same as with SAXMachine
+    def self.add_common_feed_entry_element(element_tag, options = {})
+      call_on_each_feed_entry :element, element_tag, options
+    end
+    
+    # Makes all registered entry types look for the passed in elements to parse.
+    # This is actually just a call to element (a SAXMachine call) in the class.
+    #
+    # === Parameters
+    # [element_tag<String>]
+    # [options<Hash>] Valid keys are same as with SAXMachine
+    def self.add_common_feed_entry_elements(element_tag, options = {})
+      call_on_each_feed_entry :elements, element_tag, options
+    end
+
+    # Call a method on all feed entries classes.
+    #
+    # === Parameters
+    # [method<Symbol>] The method name
+    # [parameters<Array>] The method parameters
+    def self.call_on_each_feed_entry(method, *parameters)
+      feed_classes.each do |k|
+        # iterate on the collections defined in the sax collection
+        k.sax_config.collection_elements.each_value do |vl|
+          # vl is a list of CollectionConfig mapped to an attribute name
+          # we'll look for the one set as 'entries' and add the new element
+          vl.find_all{|v| (v.accessor == 'entries') && (v.data_class.class == Class)}.each do |v|
+              v.data_class.send(method, *parameters)
+          end
+        end
+      end
+    end
+
+    # Setup curl from options.
+    # Possible parameters:
+    # * :user_agent          - overrides the default user agent.
+    # * :compress            - any value to enable compression
+    # * :http_authentication - array containing http authentication parameters
+    # * :proxy_url           - proxy url
+    # * :proxy_port          - proxy port
+    # * :max_redirects       - max number of redirections
+    # * :timeout             - timeout
+    def self.setup_easy curl, options
+      curl.headers["Accept-encoding"]   = 'gzip, deflate' if options.has_key?(:compress)
+      curl.headers["User-Agent"]        = (options[:user_agent] || USER_AGENT)
+
+      curl.userpwd = options[:http_authentication].join(':') if options.has_key?(:http_authentication)
+      curl.proxy_url = options[:proxy_url] if options.has_key?(:proxy_url)
+      curl.proxy_port = options[:proxy_port] if options.has_key?(:proxy_port)
+      curl.max_redirects = options[:max_redirects] if options[:max_redirects]
+      curl.timeout = options[:timeout] if options[:timeout]
+      curl.ssl_verify_host = options[:ssl_verify_host] if options.has_key?(:ssl_verify_host)
+
+      curl.follow_location = true
+    end
+
+    # Fetches and returns the raw XML for each URL provided.
+    #
+    # === Parameters
+    # [urls<String> or <Array>] A single feed URL, or an array of feed URLs.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 :if_modified_since - Time object representing when the feed was last updated.
+    #                 :if_none_match - String that's normally an etag for the request that was stored previously.
+    #                 :on_success - Block that gets executed after a successful request.
+    #                 :on_failure - Block that gets executed after a failed request.
+    #                 * all parameters defined in setup_easy
+    # === Returns
+    # A String of XML if a single URL is passed.
+    # 
+    # A Hash if multiple URL's are passed. The key will be the URL, and the value the XML.
+    def self.fetch_raw(urls, options = {})
+      url_queue = [*urls]
+      multi = Curl::Multi.new
+      responses = {}
+      url_queue.each do |url|
+        easy = Curl::Easy.new(url) do |curl|
+          setup_easy curl, options
+
+          curl.headers["If-Modified-Since"] = options[:if_modified_since].httpdate if options.has_key?(:if_modified_since)
+          curl.headers["If-None-Match"]     = options[:if_none_match] if options.has_key?(:if_none_match)
+
+          curl.on_success do |c|
+            responses[url] = decode_content(c)
+          end
+          curl.on_failure do |c, err|
+            responses[url] = c.response_code
+          end
+        end
+        multi.add(easy)
+      end
+
+      multi.perform
+      urls.is_a?(String) ? responses.values.first : responses
+    end
+
+    # Fetches and returns the parsed XML for each URL provided.
+    #
+    # === Parameters
+    # [urls<String> or <Array>] A single feed URL, or an array of feed URLs.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    # * :user_agent - String that overrides the default user agent.
+    # * :if_modified_since - Time object representing when the feed was last updated.
+    # * :if_none_match - String, an etag for the request that was stored previously.
+    # * :on_success - Block that gets executed after a successful request.
+    # * :on_failure - Block that gets executed after a failed request.
+    # === Returns
+    # A Feed object if a single URL is passed.
+    #
+    # A Hash if multiple URL's are passed. The key will be the URL, and the value the Feed object.
+    def self.fetch_and_parse(urls, options = {})
+      url_queue = [*urls]
+      multi = Curl::Multi.new
+      responses = {}
+      
+      # I broke these down so I would only try to do 30 simultaneously because
+      # I was getting weird errors when doing a lot. As one finishes it pops another off the queue.
+      url_queue.slice!(0, 30).each do |url|
+        add_url_to_multi(multi, url, url_queue, responses, options)
+      end
+ 
+      multi.perform
+      return urls.is_a?(String) ? responses.values.first : responses
+    end
+
+    # Decodes the XML document if it was compressed.
+    #
+    # === Parameters
+    # [curl_request<Curl::Easy>] The Curl::Easy response object from the request.
+    # === Returns
+    # A decoded string of XML.
+    def self.decode_content(c)
+      if c.header_str.match(/Content-Encoding: gzip/i)
+        begin
+          gz =  Zlib::GzipReader.new(StringIO.new(c.body_str))
+          xml = gz.read
+          gz.close
+        rescue Zlib::GzipFile::Error 
+          # Maybe this is not gzipped?
+          xml = c.body_str
+        end
+      elsif c.header_str.match(/Content-Encoding: deflate/i)
+        xml = Zlib::Inflate.inflate(c.body_str)
+      else
+        xml = c.body_str
+      end
+
+      xml
+    end
+
+    # Updates each feed for each Feed object provided.
+    #
+    # === Parameters
+    # [feeds<Feed> or <Array>] A single feed object, or an array of feed objects.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 * :on_success - Block that gets executed after a successful request.
+    #                 * :on_failure - Block that gets executed after a failed request.
+    #                 * all parameters defined in setup_easy
+    # === Returns
+    # A updated Feed object if a single URL is passed.
+    #
+    # A Hash if multiple Feeds are passed. The key will be the URL, and the value the updated Feed object.
+    def self.update(feeds, options = {})
+      feed_queue = [*feeds]
+      multi = Curl::Multi.new
+      responses = {}
+      
+      feed_queue.slice!(0, 30).each do |feed|
+        add_feed_to_multi(multi, feed, feed_queue, responses, options)
+      end
+    
+      multi.perform
+      responses.is_a?(Array)? responses.values : responses.values.first
+    end
+    
+    # An abstraction for adding a feed by URL to the passed Curb::multi stack.
+    #
+    # === Parameters
+    # [multi<Curl::Multi>] The Curl::Multi object that the request should be added too.
+    # [url<String>] The URL of the feed that you would like to be fetched.
+    # [url_queue<Array>] An array of URLs that are queued for request.
+    # [responses<Hash>] Existing responses that you want the response from the request added to.
+    # [feeds<String> or <Array>] A single feed object, or an array of feed objects.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 * :on_success - Block that gets executed after a successful request.
+    #                 * :on_failure - Block that gets executed after a failed request.
+    #                 * all parameters defined in setup_easy
+    # === Returns
+    # The updated Curl::Multi object with the request details added to it's stack.
+    def self.add_url_to_multi(multi, url, url_queue, responses, options)
+      easy = Curl::Easy.new(url) do |curl|
+        setup_easy curl, options
+        curl.headers["If-Modified-Since"] = options[:if_modified_since].httpdate if options.has_key?(:if_modified_since)
+        curl.headers["If-None-Match"]     = options[:if_none_match] if options.has_key?(:if_none_match)
+
+        curl.on_success do |c|
+          add_url_to_multi(multi, url_queue.shift, url_queue, responses, options) unless url_queue.empty?
+          xml = decode_content(c)
+          klass = determine_feed_parser_for_xml(xml)
+          
+          if klass
+            begin
+              feed = klass.parse(xml, Proc.new{|message| warn "Error while parsing [#{url}] #{message}" })
+              feed.feed_url = c.last_effective_url
+              feed.etag = etag_from_header(c.header_str)
+              feed.last_modified = last_modified_from_header(c.header_str)
+              responses[url] = feed
+              options[:on_success].call(url, feed) if options.has_key?(:on_success)
+            rescue Exception => e
+              options[:on_failure].call(url, c.response_code, c.header_str, c.body_str) if options.has_key?(:on_failure)
+            end
+          else
+            # puts "Error determining parser for #{url} - #{c.last_effective_url}"
+            # raise NoParserAvailable.new("no valid parser for content.") (this would unfortunately fail the whole 'multi', so it's not really usable)
+            options[:on_failure].call(url, c.response_code, c.header_str, c.body_str) if options.has_key?(:on_failure)
+          end
+        end
+        
+        curl.on_failure do |c, err|
+          add_url_to_multi(multi, url_queue.shift, url_queue, responses, options) unless url_queue.empty?
+          responses[url] = c.response_code
+          if c.response_code == 304 # it's not modified. this isn't an error condition
+            options[:on_success].call(url, nil) if options.has_key?(:on_success)
+          else
+            options[:on_failure].call(url, c.response_code, c.header_str, c.body_str) if options.has_key?(:on_failure)
+          end
+        end
+      end
+      multi.add(easy)
+    end
+
+    # An abstraction for adding a feed by a Feed object to the passed Curb::multi stack.
+    #
+    # === Parameters
+    # [multi<Curl::Multi>] The Curl::Multi object that the request should be added too.
+    # [feed<Feed>] A feed object that you would like to be fetched.
+    # [url_queue<Array>] An array of feed objects that are queued for request.
+    # [responses<Hash>] Existing responses that you want the response from the request added to.
+    # [feeds<String>] or <Array> A single feed object, or an array of feed objects.
+    # [options<Hash>] Valid keys for this argument as as followed:
+    #                 * :on_success - Block that gets executed after a successful request.
+    #                 * :on_failure - Block that gets executed after a failed request.
+    #                 * all parameters defined in setup_easy
+    # === Returns
+    # The updated Curl::Multi object with the request details added to it's stack.
+    def self.add_feed_to_multi(multi, feed, feed_queue, responses, options) 
+      easy = Curl::Easy.new(feed.feed_url) do |curl|
+        setup_easy curl, options
+        curl.headers["If-Modified-Since"] = feed.last_modified.httpdate if feed.last_modified
+        curl.headers["If-Modified-Since"] = options[:if_modified_since] if options[:if_modified_since] && (!feed.last_modified || (Time.parse(options[:if_modified_since].to_s) > feed.last_modified))
+        curl.headers["If-None-Match"]     = feed.etag if feed.etag
+
+        curl.on_success do |c|
+          begin
+            add_feed_to_multi(multi, feed_queue.shift, feed_queue, responses, options) unless feed_queue.empty?
+            updated_feed = Feed.parse(c.body_str){ |message| warn "Error while parsing [#{feed.feed_url}] #{message}" }
+            updated_feed.feed_url = c.last_effective_url
+            updated_feed.etag = etag_from_header(c.header_str)
+            updated_feed.last_modified = last_modified_from_header(c.header_str)
+            feed.update_from_feed(updated_feed)
+            responses[feed.feed_url] = feed
+            options[:on_success].call(feed) if options.has_key?(:on_success)
+          rescue Exception => e
+            options[:on_failure].call(feed, c.response_code, c.header_str, c.body_str) if options.has_key?(:on_failure)
+          end
+        end
+
+        curl.on_failure do |c, err|
+          add_feed_to_multi(multi, feed_queue.shift, feed_queue, responses, options) unless feed_queue.empty?
+          response_code = c.response_code
+          if response_code == 304 # it's not modified. this isn't an error condition
+            responses[feed.feed_url] = feed
+            options[:on_success].call(feed) if options.has_key?(:on_success)
+          else
+            responses[feed.url] = c.response_code
+            options[:on_failure].call(feed, c.response_code, c.header_str, c.body_str) if options.has_key?(:on_failure)
+          end
+        end
+      end
+      multi.add(easy)
+    end
+
+    # Determines the etag from the request headers.
+    # 
+    # === Parameters
+    # [header<String>] Raw request header returned from the request
+    # === Returns
+    # A string of the etag or nil if it cannot be found in the headers.
+    def self.etag_from_header(header)
+      header =~ /.*ETag:\s(.*)\r/
+      $1
+    end
+
+    # Determines the last modified date from the request headers.
+    #
+    # === Parameters
+    # [header<String>] Raw request header returned from the request
+    # === Returns
+    # A Time object of the last modified date or nil if it cannot be found in the headers.
+    def self.last_modified_from_header(header)
+      header =~ /.*Last-Modified:\s(.*)\r/
+      Time.parse($1) if $1
+    end
+  end
+end