seanwalbran-feedzirra 0.1.2

data/lib/feedzirra/parser/rss_feed_burner_entry.rb

@@ -0,0 +1,40 @@
+module Feedzirra
+
+  module Parser
+    # Parser for dealing with RDF feed entries.
+    class RSSFeedBurnerEntry
+        include SAXMachine
+        include FeedEntryUtilities
+
+        element :title
+
+        element :"feedburner:origLink", :as => :url
+        element :link, :as => :url
+
+        element :"dc:creator", :as => :author
+        element :author, :as => :author
+        element :"content:encoded", :as => :content
+        element :description, :as => :summary
+
+        element :pubDate, :as => :published
+        element :pubdate, :as => :published
+        element :"dc:date", :as => :published
+        element :"dc:Date", :as => :published
+        element :"dcterms:created", :as => :published
+
+
+        element :"dcterms:modified", :as => :updated
+        element :issued, :as => :published
+        elements :category, :as => :categories
+
+        element :guid, :as => :entry_id
+
+        def url
+          @url || @link
+        end
+      
+    end
+
+  end
+
+end

data/lib/feedzirra/version.rb

@@ -0,0 +1,3 @@
+module Feedzirra
+  VERSION = '0.1.2'
+end

data/spec/benchmarks/feed_benchmarks.rb

@@ -0,0 +1,98 @@
+# this is some spike code to compare the speed of different methods for performing
+# multiple feed fetches
+require 'rubygems'
+require 'curb'
+require 'active_support'
+
+require 'net/http'
+require 'uri'
+
+require 'benchmark'
+include Benchmark
+
+GET_COUNT = 1
+urls = ["http://www.pauldix.net"] * GET_COUNT
+
+
+benchmark do |t|
+  t.report("taf2-curb") do
+    multi = Curl::Multi.new
+    urls.each do |url|
+      easy = Curl::Easy.new(url) do |curl|
+        curl.headers["User-Agent"] = "feedzirra"
+    #    curl.headers["If-Modified-Since"] = Time.now.httpdate
+    #    curl.headers["If-None-Match"] = "ziEyTl4q9GH04BR4jgkImd0GvSE"
+        curl.follow_location = true
+        curl.on_success do |c|
+    #      puts c.header_str.inspect
+#          puts c.response_code
+    #      puts c.body_str.slice(0, 500)
+        end
+        curl.on_failure do |c|
+          puts "**** #{c.response_code}"
+        end
+      end
+      multi.add(easy)
+    end
+
+    multi.perform
+  end
+
+  t.report("nethttp") do
+    urls.each do |url|
+      res = Net::HTTP.get(URI.parse(url))
+#      puts res.slice(0, 500)
+    end
+  end
+  
+  require 'rfuzz/session'
+  include RFuzz 
+  t.report("rfuzz") do
+    GET_COUNT.times do
+      http = HttpClient.new("www.pauldix.net", 80)
+      response = http.get("/")
+      if response.http_status != "200" 
+        puts "***** #{response.http_status}"
+      else
+#        puts response.http_status
+  #      puts response.http_body.slice(0, 500)
+      end
+    end
+  end
+  
+  require 'eventmachine'
+  t.report("eventmachine") do
+    counter = GET_COUNT
+    EM.run do
+      GET_COUNT.times do
+        http = EM::Protocols::HttpClient2.connect("www.pauldix.net", 80)
+        request = http.get("/")
+        request.callback do
+#          puts request.status
+#          puts request.content.slice(0, 500)
+          counter -= 1
+          EM.stop if counter == 0
+        end
+      end
+    end
+  end
+  
+  
+  require 'curl-multi'
+  t.report("curl multi") do
+    multi = Curl::Multi.new
+    urls.each do |url|
+      on_failure = lambda do |ex|
+        puts "****** Failed to retrieve #{url}"
+      end
+
+      on_success = lambda do |body|
+#        puts "got #{url}"
+#        puts body.slice(0, 500)
+      end
+      multi.get(url, on_success, on_failure)
+    end
+
+    multi.select([], []) while multi.size > 0
+  end
+end

data/spec/benchmarks/feedzirra_benchmarks.rb

@@ -0,0 +1,40 @@
+require File.dirname(__FILE__) + '/../../lib/feedzirra.rb'
+require 'rfeedparser'
+require 'feed-normalizer'
+require 'open-uri'
+
+require 'benchmark'
+include Benchmark
+
+iterations = 10
+urls = File.readlines(File.dirname(__FILE__) + "/../sample_feeds/successful_feed_urls.txt").slice(0, 20)
+puts "benchmarks on #{urls.size} feeds"
+puts "************************************"
+benchmark do |t|    
+  t.report("feedzirra") do
+    iterations.times do
+      Feedzirra::Feed.fetch_and_parse(urls, :on_success => lambda { |url, feed| $stdout.print '.'; $stdout.flush })
+    end
+  end
+
+  t.report("rfeedparser") do
+    iterations.times do
+      urls.each do |url|
+        feed = FeedParser.parse(url)
+        $stdout.print '.'
+        $stdout.flush
+      end
+    end
+  end
+
+  t.report("feed-normalizer") do
+    iterations.times do
+      urls.each do |url|
+        # have to use the :force option to make feed-normalizer parse an atom feed
+        feed = FeedNormalizer::FeedNormalizer.parse(open(url), :force_parser => FeedNormalizer::SimpleRssParser)
+        $stdout.print '.'
+        $stdout.flush
+      end
+    end
+  end
+end

data/spec/benchmarks/fetching_benchmarks.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require File.dirname(__FILE__) + '/../../lib/feedzirra.rb'
+
+require 'open-uri'
+
+require 'benchmark'
+include Benchmark
+
+iterations = 10
+urls = File.readlines(File.dirname(__FILE__) + "/../sample_feeds/successful_feed_urls.txt").slice(0, 20)
+puts "benchmarks on #{urls.size} feeds"
+puts "************************************"
+benchmark do |t|    
+  t.report("feedzirra open uri") do
+    iterations.times do
+      urls.each do |url|
+        Feedzirra::Feed.parse(open(url, "User-Agent" => "feedzirra http://github.com/pauldix/feedzirra/tree/master").read)
+        $stdout.print '.'; $stdout.flush
+      end
+    end
+  end
+
+  t.report("feedzirra fetch and parse") do
+    iterations.times do
+      Feedzirra::Feed.fetch_and_parse(urls, :on_success => lambda { |url, feed| $stdout.print '.'; $stdout.flush })
+    end
+  end  
+end

data/spec/benchmarks/parsing_benchmark.rb

@@ -0,0 +1,30 @@
+require File.dirname(__FILE__) + '/../../lib/feedzirra.rb'
+require 'rfeedparser'
+require 'feed-normalizer'
+
+require 'benchmark'
+include Benchmark
+
+iterations = 50
+xml = File.read(File.dirname(__FILE__) + '/../sample_feeds/PaulDixExplainsNothing.xml')
+
+benchmark do |t|    
+  t.report("feedzirra") do
+    iterations.times do
+      Feedzirra::Feed.parse(xml)
+    end
+  end
+
+  t.report("rfeedparser") do
+    iterations.times do
+      FeedParser.parse(xml)
+    end
+  end
+
+  t.report("feed-normalizer") do
+    iterations.times do
+      # have to use the :force option to make feed-normalizer parse an atom feed
+      FeedNormalizer::FeedNormalizer.parse(xml, :force_parser => FeedNormalizer::SimpleRssParser)
+    end
+  end
+end

data/spec/benchmarks/updating_benchmarks.rb

@@ -0,0 +1,33 @@
+require 'rubygems'
+require File.dirname(__FILE__) + '/../../lib/feedzirra.rb'
+
+require 'benchmark'
+include Benchmark
+
+urls = File.readlines(File.dirname(__FILE__) + "/../sample_feeds/successful_feed_urls.txt")
+puts "benchmarks on #{urls.size} feeds"
+puts "************************************"
+benchmark do |t|
+  feeds = {}
+  t.report("feedzirra fetch and parse") do
+    feeds = Feedzirra::Feed.fetch_and_parse(urls, 
+      :on_success => lambda { |url, feed| $stdout.print '.'; $stdout.flush },
+      :on_failure => lambda {|url, response_code, header, body| puts "#{response_code} ERROR on #{url}"})
+  end
+  
+  # curb caches the dns lookups for 60 seconds. to make things fair we have to wait for the cache to expire
+  puts "sleeping to wait for dns cache to clear"
+  65.times {$stdout.print('.'); sleep(1)}
+  puts "done"
+  
+  updated_feeds = []
+  t.report("feedzirra update") do
+    updated_feeds = Feedzirra::Feed.update(feeds.values.reject {|f| f.class == Fixnum}, 
+      :on_success => lambda {|feed| $stdout.print '.'; $stdout.flush},
+      :on_failure => lambda {|feed, response_code, header, body| puts "#{response_code} ERROR on #{feed.feed_url}"})
+  end
+  
+  updated_feeds.each do |feed|
+    puts feed.feed_url if feed.updated?
+  end
+end

data/spec/feedzirra/feed_entry_utilities_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+describe Feedzirra::FeedUtilities do
+  before(:each) do
+    @klass = Class.new do
+      include Feedzirra::FeedEntryUtilities
+    end
+  end
+  
+  describe "handling dates" do
+    it "should parse an ISO 8601 formatted datetime into Time" do
+      time = @klass.new.parse_datetime("2008-02-20T8:05:00-010:00")
+      time.class.should == Time
+      time.to_s.should == "Wed Feb 20 18:05:00 UTC 2008"
+    end
+  end
+  
+  describe "sanitizing" do
+    before(:each) do
+      @feed = Feedzirra::Feed.parse(sample_atom_feed)
+      @entry = @feed.entries.first
+    end
+    
+    it "should provide a sanitized title" do
+      new_title = "<script>this is not safe</script>" + @entry.title
+      @entry.title = new_title
+      @entry.title.sanitize.should == Loofah.scrub_fragment(new_title, :prune).to_s
+    end
+    
+    it "should sanitize content in place" do
+      new_content = "<script>" + @entry.content
+      @entry.content = new_content.dup
+      @entry.content.sanitize!.should == Loofah.scrub_fragment(new_content, :prune).to_s
+      @entry.content.should == Loofah.scrub_fragment(new_content, :prune).to_s
+    end
+    
+    it "should sanitize things in place" do
+      @entry.title   += "<script>"
+      @entry.author  += "<script>"
+      @entry.content += "<script>"
+
+      cleaned_title   = Loofah.scrub_fragment(@entry.title, :prune).to_s
+      cleaned_author  = Loofah.scrub_fragment(@entry.author, :prune).to_s
+      cleaned_content = Loofah.scrub_fragment(@entry.content, :prune).to_s
+      
+      @entry.sanitize!
+      @entry.title.should   == cleaned_title
+      @entry.author.should  == cleaned_author
+      @entry.content.should == cleaned_content
+    end
+  end
+end

data/spec/feedzirra/feed_spec.rb

@@ -0,0 +1,593 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Feedzirra::Feed do
+
+  describe "#add_common_feed_element" do
+    before(:all) do
+      Feedzirra::Feed.add_common_feed_element("generator")
+    end
+
+    it "should parse the added element out of Atom feeds" do
+      Feedzirra::Feed.parse(sample_wfw_feed).generator.should == "TypePad"
+    end
+
+    it "should parse the added element out of Atom Feedburner feeds" do
+      Feedzirra::Parser::Atom.new.should respond_to(:generator)
+    end
+
+    it "should parse the added element out of RSS feeds" do
+      Feedzirra::Parser::RSS.new.should respond_to(:generator)
+    end
+  end
+
+  describe "#add_common_feed_entry_element" do
+    before(:all) do
+      Feedzirra::Feed.add_common_feed_entry_element("wfw:commentRss", :as => :comment_rss)
+    end
+    
+    it "should parse the added element out of Atom feeds entries" do
+      Feedzirra::Feed.parse(sample_wfw_feed).entries.first.comment_rss.should == "this is the new val"
+    end
+    
+    it "should parse the added element out of Atom Feedburner feeds entries" do
+      Feedzirra::Parser::AtomEntry.new.should respond_to(:comment_rss)
+    end
+    
+    it "should parse the added element out of RSS feeds entries" do
+      Feedzirra::Parser::RSSEntry.new.should respond_to(:comment_rss)
+    end
+  end
+  
+  describe "#parse" do # many of these tests are redundant with the specific feed type tests, but I put them here for completeness
+    context "when there's an available parser" do
+      it "should parse an rdf feed" do
+        feed = Feedzirra::Feed.parse(sample_rdf_feed)
+        feed.title.should == "HREF Considered Harmful"
+        feed.entries.first.published.to_s.should == "Tue Sep 02 19:50:07 UTC 2008"
+        feed.entries.size.should == 10
+      end
+
+      it "should parse an rss feed" do
+        feed = Feedzirra::Feed.parse(sample_rss_feed)
+        feed.title.should == "Tender Lovemaking"
+        feed.entries.first.published.to_s.should == "Thu Dec 04 17:17:49 UTC 2008"
+        feed.entries.size.should == 10
+      end
+
+      it "should parse an atom feed" do
+        feed = Feedzirra::Feed.parse(sample_atom_feed)
+        feed.title.should == "Amazon Web Services Blog"
+        feed.entries.first.published.to_s.should == "Fri Jan 16 18:21:00 UTC 2009"
+        feed.entries.size.should == 10
+      end
+
+      it "should parse an feedburner atom feed" do
+        feed = Feedzirra::Feed.parse(sample_feedburner_atom_feed)
+        feed.title.should == "Paul Dix Explains Nothing"
+        feed.entries.first.published.to_s.should == "Thu Jan 22 15:50:22 UTC 2009"
+        feed.entries.size.should == 5
+      end      
+
+      it "should parse an itunes feed as a standard RSS feed" do
+        feed = Feedzirra::Feed.parse(sample_itunes_feed)
+        feed.title.should == "All About Everything"
+        feed.entries.first.published.should == Time.parse("Wed, 15 Jun 2005 19:00:00 GMT")
+        
+        # Since the commit 621957879, iTunes feeds will be parsed as standard RSS, so this
+        # entry should now not have a method for itunes_author.
+        feed.entries.first.should_not respond_to(:itunes_author)
+        feed.entries.size.should == 3
+      end
+    end
+
+    context "when there's no available parser" do
+      it "raises Feedzirra::NoParserAvailable" do
+        proc {
+          Feedzirra::Feed.parse("I'm an invalid feed")
+        }.should raise_error(Feedzirra::NoParserAvailable)
+      end
+    end
+
+    it "should parse an feedburner rss feed" do
+      feed = Feedzirra::Feed.parse(sample_rss_feed_burner_feed)
+      feed.title.should == "TechCrunch"
+      feed.entries.first.published.to_s.should == "Wed Nov 02 17:25:27 UTC 2011"
+      feed.entries.size.should == 20
+    end
+  end
+
+  describe "#determine_feed_parser_for_xml" do
+    it "should return the Feedzirra::Parser::Atom class for an atom feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_atom_feed).should == Feedzirra::Parser::Atom
+    end
+
+    it "should return the Feedzirra::Parser::AtomFeedBurner class for an atom feedburner feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_feedburner_atom_feed).should == Feedzirra::Parser::AtomFeedBurner
+    end
+
+    it "should return the Feedzirra::Parser::RSS class for an rdf/rss 1.0 feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_rdf_feed).should == Feedzirra::Parser::RSS
+    end
+
+    it "should return the Feedzirra::Parser::RSSFeedBurner class for an rss feedburner feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_rss_feed_burner_feed).should == Feedzirra::Parser::RSSFeedBurner
+    end
+
+    it "should return the Feedzirra::Parser::RSS object for an rss 2.0 feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_rss_feed).should == Feedzirra::Parser::RSS
+    end
+
+    it "should return a Feedzirra::Parser::RSS object for an itunes feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_itunes_feed).should == Feedzirra::Parser::RSS
+    end
+
+  end
+
+  describe "when adding feed types" do
+    it "should prioritize added types over the built in ones" do
+      feed_text = "Atom asdf"
+      Feedzirra::Parser::Atom.stub!(:able_to_parse?).and_return(true)
+      new_feed_type = Class.new do
+        def self.able_to_parse?(val)
+          true
+        end
+      end
+      
+      new_feed_type.should be_able_to_parse(feed_text)
+      Feedzirra::Feed.add_feed_class(new_feed_type)
+      Feedzirra::Feed.determine_feed_parser_for_xml(feed_text).should == new_feed_type
+
+      # this is a hack so that this doesn't break the rest of the tests
+      Feedzirra::Feed.feed_classes.reject! {|o| o == new_feed_type }
+    end
+  end
+
+  describe '#etag_from_header' do
+    before(:each) do
+      @header = "HTTP/1.0 200 OK\r\nDate: Thu, 29 Jan 2009 03:55:24 GMT\r\nServer: Apache\r\nX-FB-Host: chi-write6\r\nLast-Modified: Wed, 28 Jan 2009 04:10:32 GMT\r\nETag: ziEyTl4q9GH04BR4jgkImd0GvSE\r\nP3P: CP=\"ALL DSP COR NID CUR OUR NOR\"\r\nConnection: close\r\nContent-Type: text/xml;charset=utf-8\r\n\r\n"
+    end
+
+    it "should return the etag from the header if it exists" do
+      Feedzirra::Feed.etag_from_header(@header).should == "ziEyTl4q9GH04BR4jgkImd0GvSE"
+    end
+
+    it "should return nil if there is no etag in the header" do
+      Feedzirra::Feed.etag_from_header("foo").should be_nil
+    end
+
+  end
+
+  describe '#last_modified_from_header' do
+    before(:each) do
+      @header = "HTTP/1.0 200 OK\r\nDate: Thu, 29 Jan 2009 03:55:24 GMT\r\nServer: Apache\r\nX-FB-Host: chi-write6\r\nLast-Modified: Wed, 28 Jan 2009 04:10:32 GMT\r\nETag: ziEyTl4q9GH04BR4jgkImd0GvSE\r\nP3P: CP=\"ALL DSP COR NID CUR OUR NOR\"\r\nConnection: close\r\nContent-Type: text/xml;charset=utf-8\r\n\r\n"
+    end
+
+    it "should return the last modified date from the header if it exists" do
+      Feedzirra::Feed.last_modified_from_header(@header).should == Time.parse("Wed, 28 Jan 2009 04:10:32 GMT")
+    end
+
+    it "should return nil if there is no last modified date in the header" do
+      Feedzirra::Feed.last_modified_from_header("foo").should be_nil
+    end
+  end
+
+  describe "fetching feeds" do
+    before(:each) do
+      @paul_feed = { :xml => load_sample("PaulDixExplainsNothing.xml"), :url => "http://feeds.feedburner.com/PaulDixExplainsNothing" }
+      @trotter_feed = { :xml => load_sample("TrotterCashionHome.xml"), :url => "http://feeds2.feedburner.com/trottercashion" }
+    end
+
+    describe "#fetch_raw" do
+      before(:each) do
+        @cmock = stub('cmock', :header_str => '', :body_str => @paul_feed[:xml] )
+        @multi = stub('curl_multi', :add => true, :perform => true)
+        @curl_easy = stub('curl_easy')
+        @curl = stub('curl', :headers => {}, :follow_location= => true, :on_failure => true)
+        @curl.stub!(:on_success).and_yield(@cmock)
+        
+        Curl::Multi.stub!(:new).and_return(@multi)
+        Curl::Easy.stub!(:new).and_yield(@curl).and_return(@curl_easy)
+      end
+
+      it "should set user agent if it's passed as an option" do
+        Feedzirra::Feed.fetch_raw(@paul_feed[:url], :user_agent => 'Custom Useragent')
+        @curl.headers['User-Agent'].should == 'Custom Useragent'
+      end
+
+      it "should set user agent to default if it's not passed as an option" do
+        Feedzirra::Feed.fetch_raw(@paul_feed[:url])
+        @curl.headers['User-Agent'].should == Feedzirra::Feed::USER_AGENT
+      end
+      
+      it "should set if modified since as an option if passed" do
+        Feedzirra::Feed.fetch_raw(@paul_feed[:url], :if_modified_since => Time.parse("Wed, 28 Jan 2009 04:10:32 GMT"))
+        @curl.headers["If-Modified-Since"].should == 'Wed, 28 Jan 2009 04:10:32 GMT'
+      end
+
+      it "should set if none match as an option if passed" do
+        Feedzirra::Feed.fetch_raw(@paul_feed[:url], :if_none_match => 'ziEyTl4q9GH04BR4jgkImd0GvSE')
+        @curl.headers["If-None-Match"].should == 'ziEyTl4q9GH04BR4jgkImd0GvSE'
+      end
+      
+      it 'should set userpwd for http basic authentication if :http_authentication is passed' do
+        @curl.should_receive(:userpwd=).with('username:password')
+        Feedzirra::Feed.fetch_raw(@paul_feed[:url], :http_authentication => ['username', 'password'])
+      end
+
+      it 'should set accepted encodings' do
+        Feedzirra::Feed.fetch_raw(@paul_feed[:url], :compress => true)
+        @curl.headers["Accept-encoding"].should == 'gzip, deflate'
+      end
+
+      it "should return raw xml" do
+        Feedzirra::Feed.fetch_raw(@paul_feed[:url]).should =~ /^#{Regexp.escape('<?xml version="1.0" encoding="UTF-8"?>')}/
+      end
+
+      it "should take multiple feed urls and return a hash of urls and response xml" do
+        multi = stub('curl_multi', :add => true, :perform => true)
+        Curl::Multi.stub!(:new).and_return(multi)
+        
+        paul_response = stub('paul_response', :header_str => '', :body_str => @paul_feed[:xml] )
+        trotter_response = stub('trotter_response', :header_str => '', :body_str => @trotter_feed[:xml] )
+
+        paul_curl = stub('paul_curl', :headers => {}, :follow_location= => true, :on_failure => true)
+        paul_curl.stub!(:on_success).and_yield(paul_response)
+
+        trotter_curl = stub('trotter_curl', :headers => {}, :follow_location= => true, :on_failure => true)
+        trotter_curl.stub!(:on_success).and_yield(trotter_response)
+        
+        Curl::Easy.should_receive(:new).with(@paul_feed[:url]).ordered.and_yield(paul_curl)
+        Curl::Easy.should_receive(:new).with(@trotter_feed[:url]).ordered.and_yield(trotter_curl)
+        
+        results = Feedzirra::Feed.fetch_raw([@paul_feed[:url], @trotter_feed[:url]])
+        results.keys.should include(@paul_feed[:url])
+        results.keys.should include(@trotter_feed[:url])
+        results[@paul_feed[:url]].should =~ /Paul Dix/
+        results[@trotter_feed[:url]].should =~ /Trotter Cashion/
+      end
+
+      it "should always return a hash when passed an array" do
+        results = Feedzirra::Feed.fetch_raw([@paul_feed[:url]])
+        results.class.should == Hash
+      end
+    end
+
+    describe "#add_url_to_multi" do
+      before(:each) do
+        @multi = Curl::Multi.get(@paul_feed[:url])
+        @multi.stub!(:add)
+        @easy_curl = Curl::Easy.new(@paul_feed[:url])
+        
+        Curl::Easy.should_receive(:new).and_yield(@easy_curl)
+      end
+
+      it "should set user agent if it's passed as an option" do
+        Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, :user_agent => 'My cool application')
+        @easy_curl.headers["User-Agent"].should == 'My cool application'
+      end
+      
+      it "should set user agent to default if it's not passed as an option" do
+        Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, {})
+        @easy_curl.headers["User-Agent"].should == Feedzirra::Feed::USER_AGENT
+      end
+      
+      it "should set if modified since as an option if passed" do
+        Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, :if_modified_since => Time.parse("Jan 25 2009 04:10:32 GMT"))
+        @easy_curl.headers["If-Modified-Since"].should == 'Sun, 25 Jan 2009 04:10:32 GMT'
+      end
+
+      it 'should set follow location to true' do
+        @easy_curl.should_receive(:follow_location=).with(true)
+        Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, {})
+      end
+      
+      it 'should set userpwd for http basic authentication if :http_authentication is passed' do
+        Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, :http_authentication => ['myusername', 'mypassword'])
+        @easy_curl.userpwd.should == 'myusername:mypassword'
+      end
+      
+      it 'should set accepted encodings' do
+        Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, {:compress => true})
+        @easy_curl.headers["Accept-encoding"].should == 'gzip, deflate'
+      end
+
+      it "should set if_none_match as an option if passed" do
+        Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, :if_none_match => 'ziEyTl4q9GH04BR4jgkImd0GvSE')
+        @easy_curl.headers["If-None-Match"].should == 'ziEyTl4q9GH04BR4jgkImd0GvSE'
+      end
+      
+      describe 'on success' do
+        before(:each) do
+          @feed = mock('feed', :feed_url= => true, :etag= => true, :last_modified= => true)
+          Feedzirra::Feed.stub!(:decode_content).and_return(@paul_feed[:xml])
+          Feedzirra::Feed.stub!(:determine_feed_parser_for_xml).and_return(Feedzirra::Parser::AtomFeedBurner)
+          Feedzirra::Parser::AtomFeedBurner.stub!(:parse).and_return(@feed)
+          Feedzirra::Feed.stub!(:etag_from_header).and_return('ziEyTl4q9GH04BR4jgkImd0GvSE')
+          Feedzirra::Feed.stub!(:last_modified_from_header).and_return('Wed, 28 Jan 2009 04:10:32 GMT')
+        end
+
+        it 'should decode the response body' do
+          Feedzirra::Feed.should_receive(:decode_content).with(@easy_curl).and_return(@paul_feed[:xml])
+          Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, {})
+          @easy_curl.on_success.call(@easy_curl)
+        end
+        
+        it 'should determine the xml parser class' do
+          Feedzirra::Feed.should_receive(:determine_feed_parser_for_xml).with(@paul_feed[:xml]).and_return(Feedzirra::Parser::AtomFeedBurner)
+          Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, {})
+          @easy_curl.on_success.call(@easy_curl)
+        end
+
+        it 'should parse the xml' do
+          Feedzirra::Parser::AtomFeedBurner.should_receive(:parse).with(@paul_feed[:xml], an_instance_of(Proc)).and_return(@feed)
+          Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, {})
+          @easy_curl.on_success.call(@easy_curl)
+        end
+        
+        describe 'when a compatible xml parser class is found' do
+          it 'should set the last effective url to the feed url' do
+            @easy_curl.should_receive(:last_effective_url).and_return(@paul_feed[:url])
+            @feed.should_receive(:feed_url=).with(@paul_feed[:url])
+            Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, {})
+            @easy_curl.on_success.call(@easy_curl)
+          end
+
+          it 'should set the etags on the feed' do
+            @feed.should_receive(:etag=).with('ziEyTl4q9GH04BR4jgkImd0GvSE')
+            Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, {})
+            @easy_curl.on_success.call(@easy_curl)
+          end
+
+          it 'should set the last modified on the feed' do
+            @feed.should_receive(:last_modified=).with('Wed, 28 Jan 2009 04:10:32 GMT')
+            Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, {})
+            @easy_curl.on_success.call(@easy_curl)
+          end
+
+          it 'should add the feed to the responses' do
+            responses = {}
+            Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], responses, {})
+            @easy_curl.on_success.call(@easy_curl)
+            
+            responses.length.should == 1
+            responses['http://feeds.feedburner.com/PaulDixExplainsNothing'].should == @feed
+          end
+          
+          it 'should call proc if :on_success option is passed' do
+            success = lambda { |url, feed| }
+            success.should_receive(:call).with(@paul_feed[:url], @feed)
+            Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, { :on_success => success })
+            @easy_curl.on_success.call(@easy_curl)
+          end
+        end
+
+        describe 'when no compatible xml parser class is found' do
+          it 'should raise a NoParserAvailable exception'
+        end
+      end
+
+      describe 'on failure' do
+        before(:each) do
+          @headers = "HTTP/1.0 404 Not Found\r\nDate: Thu, 29 Jan 2009 03:55:24 GMT\r\nServer: Apache\r\nX-FB-Host: chi-write6\r\nLast-Modified: Wed, 28 Jan 2009 04:10:32 GMT\r\n"
+          @body = 'Page could not be found.'
+
+          @easy_curl.stub!(:response_code).and_return(404)
+          @easy_curl.stub!(:header_str).and_return(@headers)
+          @easy_curl.stub!(:body_str).and_return(@body)
+        end
+
+        it 'should call proc if :on_failure option is passed' do
+          failure = lambda { |url, feed| }
+          failure.should_receive(:call).with(@paul_feed[:url], 404, @headers, @body)
+          Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, { :on_failure => failure })
+          @easy_curl.on_failure.call(@easy_curl)
+        end
+        
+        it 'should return the http code in the responses' do
+          responses = {}
+          Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], responses, {})
+          @easy_curl.on_failure.call(@easy_curl)
+
+          responses.length.should == 1
+          responses[@paul_feed[:url]].should == 404
+        end
+      end
+    end
+
+    describe "#add_feed_to_multi" do
+      before(:each) do
+        @multi = Curl::Multi.get(@paul_feed[:url])
+        @multi.stub!(:add)
+        @easy_curl = Curl::Easy.new(@paul_feed[:url])
+        @feed = Feedzirra::Feed.parse(sample_feedburner_atom_feed)
+
+        Curl::Easy.should_receive(:new).and_yield(@easy_curl)
+      end
+
+      it "should set user agent if it's passed as an option" do
+        Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, :user_agent => 'My cool application')
+        @easy_curl.headers["User-Agent"].should == 'My cool application'
+      end
+      
+      it "should set user agent to default if it's not passed as an option" do
+        Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, {})
+        @easy_curl.headers["User-Agent"].should == Feedzirra::Feed::USER_AGENT
+      end
+
+      it "should set if modified since as an option if passed" do
+        modified_time = Time.parse("Wed, 28 Jan 2009 04:10:32 GMT")
+        Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, {:if_modified_since => modified_time})
+        modified_time.should be > @feed.last_modified
+        
+        @easy_curl.headers["If-Modified-Since"].should == modified_time
+      end
+      
+      it 'should set follow location to true' do
+        @easy_curl.should_receive(:follow_location=).with(true)
+        Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, {})
+      end
+
+      it 'should set userpwd for http basic authentication if :http_authentication is passed' do
+        Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, :http_authentication => ['myusername', 'mypassword'])
+        @easy_curl.userpwd.should == 'myusername:mypassword'
+      end
+
+      it "should set if_none_match as an option if passed" do
+        @feed.etag = 'ziEyTl4q9GH04BR4jgkImd0GvSE'
+        Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, {})
+        @easy_curl.headers["If-None-Match"].should == 'ziEyTl4q9GH04BR4jgkImd0GvSE'
+      end
+
+      describe 'on success' do
+        before(:each) do
+          @new_feed = @feed.clone
+          @feed.stub!(:update_from_feed)
+          Feedzirra::Feed.stub!(:decode_content).and_return(@paul_feed[:xml])
+          Feedzirra::Feed.stub!(:determine_feed_parser_for_xml).and_return(Feedzirra::Parser::AtomFeedBurner)
+          Feedzirra::Parser::AtomFeedBurner.stub!(:parse).and_return(@new_feed)
+          Feedzirra::Feed.stub!(:etag_from_header).and_return('ziEyTl4q9GH04BR4jgkImd0GvSE')
+          Feedzirra::Feed.stub!(:last_modified_from_header).and_return('Wed, 28 Jan 2009 04:10:32 GMT')
+        end
+
+        it 'should process the next feed in the queue'
+        
+        it 'should parse the updated feed' do
+          Feedzirra::Parser::AtomFeedBurner.should_receive(:parse).and_return(@new_feed)
+          Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, {})
+          @easy_curl.on_success.call(@easy_curl)
+        end
+
+        it 'should set the last effective url to the feed url' do
+          @easy_curl.should_receive(:last_effective_url).and_return(@paul_feed[:url])
+          @new_feed.should_receive(:feed_url=).with(@paul_feed[:url])
+          Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, {})
+          @easy_curl.on_success.call(@easy_curl)
+        end
+
+        it 'should set the etags on the feed' do
+          @new_feed.should_receive(:etag=).with('ziEyTl4q9GH04BR4jgkImd0GvSE')
+          Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, {})
+          @easy_curl.on_success.call(@easy_curl)
+        end
+
+        it 'should set the last modified on the feed' do
+          @new_feed.should_receive(:last_modified=).with('Wed, 28 Jan 2009 04:10:32 GMT')
+          Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, {})
+          @easy_curl.on_success.call(@easy_curl)
+        end
+
+        it 'should add the feed to the responses' do
+          responses = {}
+          Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], responses, {})
+          @easy_curl.on_success.call(@easy_curl)
+
+          responses.length.should == 1
+          responses['http://feeds.feedburner.com/PaulDixExplainsNothing'].should == @feed
+        end
+
+        it 'should call proc if :on_success option is passed' do
+          success = lambda { |feed| }
+          success.should_receive(:call).with(@feed)
+          Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, { :on_success => success })
+          @easy_curl.on_success.call(@easy_curl)
+        end
+        
+        it 'should call update from feed on the old feed with the updated feed' do
+          @feed.should_receive(:update_from_feed).with(@new_feed)
+          Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, {})
+          @easy_curl.on_success.call(@easy_curl)
+        end
+      end
+
+      describe 'on failure' do
+        before(:each) do
+          @headers = "HTTP/1.0 404 Not Found\r\nDate: Thu, 29 Jan 2009 03:55:24 GMT\r\nServer: Apache\r\nX-FB-Host: chi-write6\r\nLast-Modified: Wed, 28 Jan 2009 04:10:32 GMT\r\n"
+          @body = 'Page could not be found.'
+
+          @easy_curl.stub!(:response_code).and_return(404)
+          @easy_curl.stub!(:header_str).and_return(@headers)
+          @easy_curl.stub!(:body_str).and_return(@body)
+        end
+
+        it 'should call on success callback if the response code is 304' do
+          success = lambda { |feed| }
+          success.should_receive(:call).with(@feed)
+          @easy_curl.should_receive(:response_code).and_return(304)
+          Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, { :on_success => success })
+          @easy_curl.on_failure.call(@easy_curl)
+        end
+        
+        it 'should return the http code in the responses' do
+          responses = {}
+          Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], responses, {})
+          @easy_curl.on_failure.call(@easy_curl)
+
+          responses.length.should == 1
+          responses['http://www.pauldix.net/'].should == 404
+        end
+      end
+    end
+
+    describe "#fetch_and_parse" do
+      it 'should initiate the fetching and parsing using multicurl'
+      it "should pass any request options through to add_url_to_multi"
+      it 'should slice the feeds into groups of thirty for processing'
+      it "should return a feed object if a single feed is passed in"
+      it "should return an return an array of feed objects if multiple feeds are passed in"
+      
+      it "should set if modified since as an option if passed" do
+        modified_time = Time.parse("Wed, 28 Jan 2009 04:10:32 GMT")
+        Feedzirra::Feed.should_receive(:add_url_to_multi).with(anything, anything, anything, anything, {:if_modified_since => modified_time}).any_number_of_times
+        
+        @feed = Feedzirra::Feed.fetch_and_parse(sample_feedburner_atom_feed, {:if_modified_since => modified_time})
+      end
+      
+    end
+
+    describe "#decode_content" do
+      before(:each) do
+        @curl_easy = mock('curl_easy', :body_str => '<xml></xml>')
+      end
+
+      it 'should decode the response body using gzip if the Content-Encoding: is gzip' do
+        @curl_easy.stub!(:header_str).and_return('Content-Encoding: gzip')
+        string_io = mock('stringio', :read => @curl_easy.body_str, :close => true)
+        StringIO.should_receive(:new).and_return(string_io)
+        Zlib::GzipReader.should_receive(:new).with(string_io).and_return(string_io)
+        Feedzirra::Feed.decode_content(@curl_easy)
+      end
+      
+      it 'should decode the response body using gzip if the Content-Encoding: is gzip even when the case is wrong' do
+        @curl_easy.stub!(:header_str).and_return('content-encoding: gzip')
+        string_io = mock('stringio', :read => @curl_easy.body_str, :close => true)
+        StringIO.should_receive(:new).and_return(string_io)
+        Zlib::GzipReader.should_receive(:new).with(string_io).and_return(string_io)
+        Feedzirra::Feed.decode_content(@curl_easy)
+      end
+
+      it 'should deflate the response body using inflate if the Content-Encoding: is deflate' do
+        @curl_easy.stub!(:header_str).and_return('Content-Encoding: deflate')
+        Zlib::Inflate.should_receive(:inflate).with(@curl_easy.body_str)
+        Feedzirra::Feed.decode_content(@curl_easy)
+      end
+
+      it 'should deflate the response body using inflate if the Content-Encoding: is deflate event if the case is wrong' do
+        @curl_easy.stub!(:header_str).and_return('content-encoding: deflate')
+        Zlib::Inflate.should_receive(:inflate).with(@curl_easy.body_str)
+        Feedzirra::Feed.decode_content(@curl_easy)
+      end
+
+      it 'should return the response body if it is not encoded' do
+        @curl_easy.stub!(:header_str).and_return('')
+        Feedzirra::Feed.decode_content(@curl_easy).should == '<xml></xml>'
+      end
+    end
+
+    describe "#update" do
+      it 'should perform the updating using multicurl'
+      it "should pass any request options through to add_feed_to_multi"
+      it "should return a feed object if a single feed is passed in"
+      it "should return an return an array of feed objects if multiple feeds are passed in" 
+    end
+  end
+end