astro-feedzirra 0.0.8.20090419 → 0.0.12

data/lib/feedzirra/parser/atom_feed_burner.rb

@@ -0,0 +1,27 @@
+module Feedzirra
+  
+  module Parser
+    # == Summary
+    # Parser for dealing with Feedburner Atom feeds.
+    #
+    # == Attributes
+    # * title
+    # * feed_url
+    # * url
+    # * entries
+    class AtomFeedBurner
+      include SAXMachine
+      include FeedUtilities
+      element :title
+      element :link, :as => :url, :value => :href, :with => {:type => "text/html"}
+      element :link, :as => :feed_url, :value => :href, :with => {:type => "application/atom+xml"}
+      elements :entry, :as => :entries, :class => AtomFeedBurnerEntry
+
+      def self.able_to_parse?(xml) #:nodoc:
+        (xml =~ /Atom/ && xml =~ /feedburner/) || false
+      end
+    end
+
+  end
+  
+end

data/lib/feedzirra/parser/atom_feed_burner_entry.rb

@@ -0,0 +1,35 @@
+module Feedzirra
+  
+  module Parser
+    # == Summary
+    # Parser for dealing with Feedburner Atom feed entries.
+    #
+    # == Attributes
+    # * title
+    # * url
+    # * author
+    # * content
+    # * summary
+    # * published
+    # * categories
+    class AtomFeedBurnerEntry
+      include SAXMachine
+      include FeedEntryUtilities
+      element :title
+      element :name, :as => :author
+      element :link, :as => :url, :value => :href, :with => {:type => "text/html", :rel => "alternate"}
+      element :"feedburner:origLink", :as => :url
+      element :summary
+      element :content
+      element :published
+      element :id
+      element :issued, :as => :published
+      element :created, :as => :published
+      element :updated
+      element :modified, :as => :updated
+      elements :category, :as => :categories, :value => :term
+    end
+
+  end
+    
+end

data/lib/feedzirra/parser/itunes_rss.rb

@@ -0,0 +1,50 @@
+module Feedzirra
+  
+  module Parser
+    # iTunes is RSS 2.0 + some apple extensions
+    # Source: http://www.apple.com/itunes/whatson/podcasts/specs.html
+    class ITunesRSS
+      include SAXMachine
+      include FeedUtilities
+
+      attr_accessor :feed_url
+
+      # RSS 2.0 elements that need including
+      element :copyright
+      element :description
+      element :language
+      element :managingEditor
+      element :title
+      element :link, :as => :url
+
+      # If author is not present use managingEditor on the channel
+      element :"itunes:author", :as => :itunes_author
+      element :"itunes:block", :as => :itunes_block
+      element :"itunes:image", :value => :href, :as => :itunes_image
+      element :"itunes:explicit", :as => :itunes_explicit
+      element :"itunes:keywords", :as => :itunes_keywords
+      # New URL for the podcast feed
+      element :"itunes:new-feed-url", :as => :itunes_new_feed_url
+      element :"itunes:subtitle", :as => :itunes_subtitle
+      # If summary is not present, use the description tag
+      element :"itunes:summary", :as => :itunes_summary
+
+      # iTunes RSS feeds can have multiple main categories...
+      # ...and multiple sub-categories per category
+      # TODO subcategories not supported correctly - they are at the same level
+      #   as the main categories
+      elements :"itunes:category", :as => :itunes_categories, :value => :text
+
+      elements :"itunes:owner", :as => :itunes_owners, :class => ITunesRSSOwner
+
+      elements :item, :as => :entries, :class => ITunesRSSItem
+
+      def self.able_to_parse?(xml)
+        xml =~ /xmlns:itunes=\"http:\/\/www.itunes.com\/dtds\/podcast-1.0.dtd\"/
+      end
+
+    end
+    
+  end
+  
+end

data/lib/feedzirra/parser/itunes_rss_item.rb

@@ -0,0 +1,32 @@
+module Feedzirra
+  
+  module Parser
+    # iTunes extensions to the standard RSS2.0 item
+    # Source: http://www.apple.com/itunes/whatson/podcasts/specs.html
+    class ITunesRSSItem
+      include SAXMachine
+      include FeedEntryUtilities
+      element :author
+      element :guid
+      element :title
+      element :link, :as => :url
+      element :description, :as => :summary
+      element :pubDate, :as => :published
+      element :"content:encoded", :as => :content
+
+      # If author is not present use author tag on the item
+      element :"itunes:author", :as => :itunes_author
+      element :"itunes:block", :as => :itunes_block
+      element :"itunes:duration", :as => :itunes_duration
+      element :"itunes:explicit", :as => :itunes_explicit
+      element :"itunes:keywords", :as => :itunes_keywords
+      element :"itunes:subtitle", :as => :itunes_subtitle
+      # If summary is not present, use the description tag
+      element :"itunes:summary", :as => :itunes_summary
+      element :enclosure, :value => :length, :as => :enclosure_length
+      element :enclosure, :value => :type, :as => :enclosure_type
+      element :enclosure, :value => :url, :as => :enclosure_url
+    end
+  end
+  
+end

data/lib/feedzirra/parser/itunes_rss_owner.rb

@@ -0,0 +1,12 @@
+module Feedzirra
+  
+  module Parser
+    class ITunesRSSOwner
+      include SAXMachine
+      include FeedUtilities
+      element :"itunes:name", :as => :name
+      element :"itunes:email", :as => :email
+    end
+  end
+  
+end

data/lib/feedzirra/parser/rss.rb

@@ -0,0 +1,28 @@
+module Feedzirra
+  
+  module Parser
+    # == Summary
+    # Parser for dealing with RSS feeds.
+    #
+    # == Attributes
+    # * title
+    # * feed_url
+    # * url
+    # * entries
+    class RSS
+      include SAXMachine
+      include FeedUtilities
+      element :title
+      element :link, :as => :url
+      elements :item, :as => :entries, :class => RSSEntry
+
+      attr_accessor :feed_url
+
+      def self.able_to_parse?(xml) #:nodoc:
+        xml =~ /\<rss|rdf/
+      end
+    end
+
+  end
+  
+end

data/lib/feedzirra/parser/rss_entry.rb

@@ -0,0 +1,40 @@
+module Feedzirra
+  
+  module Parser
+    # == Summary
+    # Parser for dealing with RDF feed entries.
+    #
+    # == Attributes
+    # * title
+    # * url
+    # * author
+    # * content
+    # * summary
+    # * published
+    # * categories
+    class RSSEntry
+      include SAXMachine
+      include FeedEntryUtilities
+      element :title
+      element :link, :as => :url
+
+      element :"dc:creator", :as => :author
+      element :"content:encoded", :as => :content
+      element :description, :as => :summary
+
+      element :pubDate, :as => :published
+      element :"dc:date", :as => :published
+      element :"dc:Date", :as => :published
+      element :"dcterms:created", :as => :published
+
+
+      element :"dcterms:modified", :as => :updated
+      element :issued, :as => :published
+      elements :category, :as => :categories
+
+      element :guid, :as => :id
+    end
+
+  end
+  
+end

data/lib/feedzirra/push_parser.rb

@@ -0,0 +1,56 @@
+module Feedzirra
+  ##
+  # Contrary to Feedzirra::Feed, Feedzirra::PushParser doesn't expect
+  # the whole document to be given in one String, but allows
+  # subsequent parsing of chunks.
+  class PushParser
+    ##
+    # How many bytes to buffer before starting to parse, helps
+    # Feedzirra's content detection.
+    BUF_MIN_THRESHOLD = 1000
+
+    ##
+    # Just resets instance variables
+    def initialize
+      @buf = ''
+      @parser = nil
+    end
+
+    ##
+    # Either buffer up til BUF_MIN_THRESHOLD or, if reached, actually
+    # parse a chunk
+    def push(chunk)
+      if @parser
+        @parser.parse(chunk)
+      else
+        @buf += chunk
+        if @buf.size > BUF_MIN_THRESHOLD
+          start_parsing
+        end
+      end
+    end
+
+    ##
+    # Really start parsing, if BUF_MIN_THRESHOLD wasn't reached yet,
+    # finalize, and return the actual parser/document
+    def finish
+      # TODO: if we haven't started yet we won't even need a
+      # PushParser
+      start_parsing unless @parser
+
+      @parser.parse_finish
+      @parser
+    end
+
+    private
+
+    def start_parsing
+      unless klass = Feed::determine_feed_parser_for_xml(@buf)
+        raise NoParserAvailable.new("No valid parser for XML.")
+      end
+      @parser = klass.new
+      @parser.parse(@buf)
+      @buf = nil
+    end
+  end
+end

data/spec/feedzirra/feed_spec.rb

@@ -11,11 +11,11 @@ describe Feedzirra::Feed do
     end
     
     it "should parse the added element out of Atom Feedburner feeds" do
-      Feedzirra::AtomEntry.new.should respond_to(:comment_rss)
+      Feedzirra::Parser::AtomEntry.new.should respond_to(:comment_rss)
     end
     
     it "should parse the added element out of RSS feeds" do
-      Feedzirra::RSSEntry.new.should respond_to(:comment_rss)
+      Feedzirra::Parser::RSSEntry.new.should respond_to(:comment_rss)
     end
   end
   
@@ -49,11 +49,14 @@ describe Feedzirra::Feed do
         feed.entries.size.should == 5
       end      
 
-      it "should parse an itunes feed" do
+      it "should parse an itunes feed as a standard RSS feed" do
         feed = Feedzirra::Feed.parse(sample_itunes_feed)
         feed.title.should == "All About Everything"
-        feed.entries.first.published.to_s.should == "Wed, 15 Jun 2005 19:00:00 GMT"
-        feed.entries.first.itunes_author.should == "John Doe"
+        feed.entries.first.published.should == Time.parse("Wed, 15 Jun 2005 19:00:00 GMT")
+        
+        # Since the commit 621957879, iTunes feeds will be parsed as standard RSS, so this
+        # entry should now not have a method for itunes_author.
+        feed.entries.first.should_not respond_to(:itunes_author)
         feed.entries.size.should == 3
       end
     end
@@ -75,28 +78,28 @@ describe Feedzirra::Feed do
   end
 
   describe "#determine_feed_parser_for_xml" do
-    it "should return the Feedzirra::Atom class for an atom feed" do
-      Feedzirra::Feed.determine_feed_parser_for_xml(sample_atom_feed).should == Feedzirra::Atom
+    it "should return the Feedzirra::Parser::Atom class for an atom feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_atom_feed).should == Feedzirra::Parser::Atom
     end
 
-    it "should return the Feedzirra::AtomFeedBurner class for an atom feedburner feed" do
-      Feedzirra::Feed.determine_feed_parser_for_xml(sample_feedburner_atom_feed).should == Feedzirra::AtomFeedBurner
+    it "should return the Feedzirra::Parser::AtomFeedBurner class for an atom feedburner feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_feedburner_atom_feed).should == Feedzirra::Parser::AtomFeedBurner
     end
 
-    it "should return the Feedzirra::RSS class for an rdf/rss 1.0 feed" do
-      Feedzirra::Feed.determine_feed_parser_for_xml(sample_rdf_feed).should == Feedzirra::RSS
+    it "should return the Feedzirra::Parser::RSS class for an rdf/rss 1.0 feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_rdf_feed).should == Feedzirra::Parser::RSS
     end
 
-    it "should return the Feedzirra::RSS class for an rss feedburner feed" do
-      Feedzirra::Feed.determine_feed_parser_for_xml(sample_rss_feed_burner_feed).should == Feedzirra::RSS
+    it "should return the Feedzirra::Parser::RSS class for an rss feedburner feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_rss_feed_burner_feed).should == Feedzirra::Parser::RSS
     end
 
-    it "should return the Feedzirra::RSS object for an rss 2.0 feed" do
-      Feedzirra::Feed.determine_feed_parser_for_xml(sample_rss_feed).should == Feedzirra::RSS
+    it "should return the Feedzirra::Parser::RSS object for an rss 2.0 feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_rss_feed).should == Feedzirra::Parser::RSS
     end
 
-    it "should return the Feedzirra::ITunesRSS object for an itunes feed" do
-      Feedzirra::Feed.determine_feed_parser_for_xml(sample_itunes_feed).should == Feedzirra::ITunesRSS
+    it "should return a Feedzirra::Parser::RSS object for an itunes feed" do
+      Feedzirra::Feed.determine_feed_parser_for_xml(sample_itunes_feed).should == Feedzirra::Parser::RSS
     end
 
   end
@@ -104,7 +107,7 @@ describe Feedzirra::Feed do
   describe "when adding feed types" do
     it "should prioritize added types over the built in ones" do
       feed_text = "Atom asdf"
-      Feedzirra::Atom.should be_able_to_parse(feed_text)
+      Feedzirra::Parser::Atom.should be_able_to_parse(feed_text)
       new_feed_type = Class.new do
         def self.able_to_parse?(val)
           true
@@ -160,7 +163,7 @@ describe Feedzirra::Feed do
         @cmock = stub('cmock', :header_str => '', :body_str => @paul_feed[:xml] )
         @multi = stub('curl_multi', :add => true, :perform => true)
         @curl_easy = stub('curl_easy')
-        @curl = stub('curl', :headers => {}, :follow_location= => true, :on_failure => true)
+        @curl = stub('curl', :headers => {}, :follow_location= => true, :on_failure => true, :timeout= => true)
         @curl.stub!(:on_success).and_yield(@cmock)
         
         Curl::Multi.stub!(:new).and_return(@multi)
@@ -193,7 +196,7 @@ describe Feedzirra::Feed do
       end
 
       it 'should set accepted encodings' do
-        Feedzirra::Feed.fetch_raw(@paul_feed[:url])
+        Feedzirra::Feed.fetch_raw(@paul_feed[:url], :compress => true)
         @curl.headers["Accept-encoding"].should == 'gzip, deflate'
       end
 
@@ -206,12 +209,12 @@ describe Feedzirra::Feed do
         Curl::Multi.stub!(:new).and_return(multi)
         
         paul_response = stub('paul_response', :header_str => '', :body_str => @paul_feed[:xml] )
-        trotter_response = stub('trotter_response', :header_str => '', :body_str => @trotter_feed[:xml] )
+        trotter_response = stub('trotter_response', :header_str => '', :body_str => @trotter_feed[:xml])
 
-        paul_curl = stub('paul_curl', :headers => {}, :follow_location= => true, :on_failure => true)
+        paul_curl = stub('paul_curl', :headers => {}, :follow_location= => true, :on_failure => true, :timeout= => true)
         paul_curl.stub!(:on_success).and_yield(paul_response)
 
-        trotter_curl = stub('trotter_curl', :headers => {}, :follow_location= => true, :on_failure => true)
+        trotter_curl = stub('trotter_curl', :headers => {}, :follow_location= => true, :on_failure => true, :timeout= => true)
         trotter_curl.stub!(:on_success).and_yield(trotter_response)
         
         Curl::Easy.should_receive(:new).with(@paul_feed[:url]).ordered.and_yield(paul_curl)
@@ -253,7 +256,7 @@ describe Feedzirra::Feed do
         Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, :if_modified_since => Time.parse("Jan 25 2009 04:10:32 GMT"))
         @easy_curl.headers["If-Modified-Since"].should == 'Sun, 25 Jan 2009 04:10:32 GMT'
       end
-
+      
       it 'should set follow location to true' do
         @easy_curl.should_receive(:follow_location=).with(true)
         Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, {})
@@ -265,7 +268,7 @@ describe Feedzirra::Feed do
       end
       
       it 'should set accepted encodings' do
-        Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, {})
+        Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, {:compress => true})
         @easy_curl.headers["Accept-encoding"].should == 'gzip, deflate'
       end
 
@@ -278,8 +281,8 @@ describe Feedzirra::Feed do
         before(:each) do
           @feed = mock('feed', :feed_url= => true, :etag= => true, :last_modified= => true)
           Feedzirra::Feed.stub!(:decode_content).and_return(@paul_feed[:xml])
-          Feedzirra::Feed.stub!(:determine_feed_parser_for_xml).and_return(Feedzirra::AtomFeedBurner)
-          Feedzirra::AtomFeedBurner.stub!(:parse).and_return(@feed)
+          Feedzirra::Feed.stub!(:determine_feed_parser_for_xml).and_return(Feedzirra::Parser::AtomFeedBurner)
+          Feedzirra::Parser::AtomFeedBurner.stub!(:parse).and_return(@feed)
           Feedzirra::Feed.stub!(:etag_from_header).and_return('ziEyTl4q9GH04BR4jgkImd0GvSE')
           Feedzirra::Feed.stub!(:last_modified_from_header).and_return('Wed, 28 Jan 2009 04:10:32 GMT')
         end
@@ -291,13 +294,13 @@ describe Feedzirra::Feed do
         end
         
         it 'should determine the xml parser class' do
-          Feedzirra::Feed.should_receive(:determine_feed_parser_for_xml).with(@paul_feed[:xml]).and_return(Feedzirra::AtomFeedBurner)
+          Feedzirra::Feed.should_receive(:determine_feed_parser_for_xml).with(@paul_feed[:xml]).and_return(Feedzirra::Parser::AtomFeedBurner)
           Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, {})
           @easy_curl.on_success.call(@easy_curl)
         end
 
         it 'should parse the xml' do
-          Feedzirra::AtomFeedBurner.should_receive(:parse).with(@paul_feed[:xml]).and_return(@feed)
+          Feedzirra::Parser::AtomFeedBurner.should_receive(:parse).with(@paul_feed[:xml]).and_return(@feed)
           Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, {})
           @easy_curl.on_success.call(@easy_curl)
         end
@@ -394,6 +397,11 @@ describe Feedzirra::Feed do
 
       it "should set if modified since as an option if passed"
       
+      it "should set timeout to the right value" do
+        @easy_curl.should_receive(:timeout=).with(8)
+        Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, {:timeout => 8})
+      end
+      
       it 'should set follow location to true' do
         @easy_curl.should_receive(:follow_location=).with(true)
         Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, {})
@@ -415,8 +423,8 @@ describe Feedzirra::Feed do
           @new_feed = @feed.clone
           @feed.stub!(:update_from_feed)
           Feedzirra::Feed.stub!(:decode_content).and_return(@paul_feed[:xml])
-          Feedzirra::Feed.stub!(:determine_feed_parser_for_xml).and_return(Feedzirra::AtomFeedBurner)
-          Feedzirra::AtomFeedBurner.stub!(:parse).and_return(@new_feed)
+          Feedzirra::Feed.stub!(:determine_feed_parser_for_xml).and_return(Feedzirra::Parser::AtomFeedBurner)
+          Feedzirra::Parser::AtomFeedBurner.stub!(:parse).and_return(@new_feed)
           Feedzirra::Feed.stub!(:etag_from_header).and_return('ziEyTl4q9GH04BR4jgkImd0GvSE')
           Feedzirra::Feed.stub!(:last_modified_from_header).and_return('Wed, 28 Jan 2009 04:10:32 GMT')
         end
@@ -424,7 +432,7 @@ describe Feedzirra::Feed do
         it 'should process the next feed in the queue'
         
         it 'should parse the updated feed' do
-          Feedzirra::AtomFeedBurner.should_receive(:parse).and_return(@new_feed)
+          Feedzirra::Parser::AtomFeedBurner.should_receive(:parse).and_return(@new_feed)
           Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, {})
           @easy_curl.on_success.call(@easy_curl)
         end