kete-feedzirra 0.0.8.1 → 0.0.16.1

data/README.textile

@@ -23,10 +23,12 @@ The final feature of Feedzirra is the ability to define custom parsing classes.
 h2. Installation
 
 For now Feedzirra exists only on github. It also has a few gem requirements that are only on github. Before you start you need to have "libcurl":http://curl.haxx.se/ and "libxml":http://xmlsoft.org/ installed. If you're on Leopard you have both. Otherwise, you'll need to grab them. Once you've got those libraries, these are the gems that get used: nokogiri, pauldix-sax-machine, taf2-curb (note that this is a fork that lives on github and not the Ruby Forge version of curb), and pauldix-feedzirra. The feedzirra gemspec has all the dependencies so you should be able to get up and running with the standard github gem install routine:
+
 <pre>
 gem sources -a http://gems.github.com # if you haven't already
 gem install pauldix-feedzirra
 </pre>
+
 <b>NOTE:</b>Some people have been reporting a few issues related to installation. First, the Ruby Forge version of curb is not what you want. It will not work. Nor will the curl-multi gem that lives on Ruby Forge. You have to get the "taf2-curb":http://github.com/taf2/curb/tree/master fork installed.
 
 If you see this error when doing a require: 
@@ -93,7 +95,7 @@ updated_feed.new_entries  # a collection of the entry objects that are newer tha
 
 # fetching multiple feeds
 feed_urls = ["http://feeds.feedburner.com/PaulDixExplainsNothing", "http://feeds.feedburner.com/trottercashion"]
-feeds = Feedzirra::Feed.fetch_and_parse(feeds_urls)
+feeds = Feedzirra::Feed.fetch_and_parse(feed_urls)
 
 # feeds is now a hash with the feed_urls as keys and the parsed feed objects as values. If an error was thrown
 # there will be a Fixnum of the http response code instead of a feed object
@@ -116,6 +118,12 @@ Feedzirra::Feed.add_common_feed_entry_element("wfw:commentRss", :as => :comment_
 # AtomEntry classes. Now you can access those in an atom feed:
 Feedzirra::Feed.parse(some_atom_xml).entries.first.comment_rss_ # => wfw:commentRss is now parsed!
 
+
+# You can also define your own parsers and add them to the ones Feedzirra knows about. Here's an example that adds 
+# ITunesRSS parsing. It's included in the library, but not part of Feedzirra by default because some of the field names 
+# differ from other classes, thus breaking normalization.
+Feedzirra::Feed.add_feed_class(ITunesRSS) # now all feeds will be checked to see if they match ITunesRSS before others
+
 # You can also access http basic auth feeds. Unfortunately, you can't get to these inside of a bulk get of a bunch of feeds.
 # You'll have to do it on its own like so:
 Feedzirra::Feed.fetch_and_parse(some_url, :http_authentication => ["myusername", "mypassword"])
@@ -151,7 +159,8 @@ This thing needs to hammer on many different feeds in the wild. I'm sure there w
 Here are some more specific TODOs.
 * Fix the iTunes parser so things are normalized again
 * Fix the Zlib deflate error
-* Fork taf2-curb and require that in feedzirra
+* Fix this error: http://github.com/inbox/70508
+* Convert to use Typhoeus instead of taf2-curb
 * Make the entries parse all link fields
 * Make a feedzirra-rails gem to integrate feedzirra seamlessly with Rails and ActiveRecord.
 * Create a super sweet DSL for defining new parsers.

data/Rakefile

@@ -20,6 +20,9 @@ def recent_specs(touched_since)
   recent_specs.uniq
 end
 
+desc "Run all the tests"
+task :default => :spec
+
 # Tasks
 Spec::Rake::SpecTask.new do |t|
   t.spec_opts = ['--options', "\"#{File.dirname(__FILE__)}/spec/spec.opts\""]

data/lib/feedzirra.rb

@@ -18,17 +18,17 @@ require 'feedzirra/feed_utilities'
 require 'feedzirra/feed_entry_utilities'
 require 'feedzirra/feed'
 
-require 'feedzirra/rss_entry'
-require 'feedzirra/itunes_rss_owner'
-require 'feedzirra/itunes_rss_item'
-require 'feedzirra/atom_entry'
-require 'feedzirra/atom_feed_burner_entry'
+require 'feedzirra/parser/rss_entry'
+require 'feedzirra/parser/itunes_rss_owner'
+require 'feedzirra/parser/itunes_rss_item'
+require 'feedzirra/parser/atom_entry'
+require 'feedzirra/parser/atom_feed_burner_entry'
 
-require 'feedzirra/rss'
-require 'feedzirra/itunes_rss'
-require 'feedzirra/atom'
-require 'feedzirra/atom_feed_burner'
+require 'feedzirra/parser/rss'
+require 'feedzirra/parser/itunes_rss'
+require 'feedzirra/parser/atom'
+require 'feedzirra/parser/atom_feed_burner'
 
 module Feedzirra
-  VERSION = "0.0.8"
+  VERSION = "0.0.16"
 end

data/lib/feedzirra/feed.rb

@@ -27,7 +27,7 @@ module Feedzirra
     # === Returns
     # The class name of the parser that can handle the XML.
     def self.determine_feed_parser_for_xml(xml)
-      start_of_doc = xml.slice(0, 1000)
+      start_of_doc = xml.slice(0, 2000)
       feed_classes.detect {|klass| klass.able_to_parse?(start_of_doc)}
     end
 
@@ -46,7 +46,7 @@ module Feedzirra
     # === Returns
     # A array of class names.
     def self.feed_classes
-      @feed_classes ||= [ITunesRSS, RSS, AtomFeedBurner, Atom]
+      @feed_classes ||= [Feedzirra::Parser::RSS, Feedzirra::Parser::AtomFeedBurner, Feedzirra::Parser::Atom]
     end
     
     # Makes all entry types look for the passed in element to parse. This is actually just a call to 
@@ -58,25 +58,11 @@ module Feedzirra
     def self.add_common_feed_entry_element(element_tag, options = {})
       # need to think of a better way to do this. will break for people who want this behavior
       # across their added classes
-      [RSSEntry, AtomFeedBurnerEntry, AtomEntry].each do |klass|
+      feed_classes.map{|k| eval("#{k}Entry") }.each do |klass|
         klass.send(:element, element_tag, options)
       end
     end
     
-    # Makes all entry types look for the passed in elements to parse. This is actually just a call to 
-    # elements (a SAXMachine call) in the class
-    #
-    # === Parameters
-    # [element_tag<String>]
-    # [options<Hash>] Valid keys are same as with SAXMachine
-    def self.add_common_feed_entry_elements(element_tag, options = {})
-      # need to think of a better way to do this. will break for people who want this behavior
-      # across their added classes
-      [RSSEntry, AtomFeedBurnerEntry, AtomEntry].each do |klass|
-        klass.send(:elements, element_tag, options)
-      end
-    end
-
     # Fetches and returns the raw XML for each URL provided.
     #
     # === Parameters
@@ -100,9 +86,12 @@ module Feedzirra
           curl.headers["User-Agent"]        = (options[:user_agent] || USER_AGENT)
           curl.headers["If-Modified-Since"] = options[:if_modified_since].httpdate if options.has_key?(:if_modified_since)
           curl.headers["If-None-Match"]     = options[:if_none_match] if options.has_key?(:if_none_match)
-          curl.headers["Accept-encoding"]   = 'gzip, deflate'
+          curl.headers["Accept-encoding"]   = 'gzip, deflate' if options.has_key?(:compress)
           curl.follow_location = true
           curl.userpwd = options[:http_authentication].join(':') if options.has_key?(:http_authentication)
+          
+          curl.max_redirects = options[:max_redirects] if options[:max_redirects]
+          curl.timeout = options[:timeout] if options[:timeout]
 
           curl.on_success do |c|
             responses[url] = decode_content(c)
@@ -115,7 +104,7 @@ module Feedzirra
       end
 
       multi.perform
-      return urls.is_a?(String) ? responses.values.first : responses
+      urls.is_a?(String) ? responses.values.first : responses
     end
 
     # Fetches and returns the parsed XML for each URL provided.
@@ -123,11 +112,11 @@ module Feedzirra
     # === Parameters
     # [urls<String> or <Array>] A single feed URL, or an array of feed URLs.
     # [options<Hash>] Valid keys for this argument as as followed:
-    #                 * :user_agent - String that overrides the default user agent.
-    #                 * :if_modified_since - Time object representing when the feed was last updated.
-    #                 * :if_none_match - String, an etag for the request that was stored previously.
-    #                 * :on_success - Block that gets executed after a successful request.
-    #                 * :on_failure - Block that gets executed after a failed request. 
+    # * :user_agent - String that overrides the default user agent.
+    # * :if_modified_since - Time object representing when the feed was last updated.
+    # * :if_none_match - String, an etag for the request that was stored previously.
+    # * :on_success - Block that gets executed after a successful request.
+    # * :on_failure - Block that gets executed after a failed request.
     # === Returns
     # A Feed object if a single URL is passed.
     #
@@ -137,12 +126,12 @@ module Feedzirra
       multi = Curl::Multi.new
       responses = {}
       
-      # I broke these down so I would only try to do 30 simultaneously because 
+      # I broke these down so I would only try to do 30 simultaneously because
       # I was getting weird errors when doing a lot. As one finishes it pops another off the queue.
       url_queue.slice!(0, 30).each do |url|
         add_url_to_multi(multi, url, url_queue, responses, options)
       end
-
+ 
       multi.perform
       return urls.is_a?(String) ? responses.values.first : responses
     end
@@ -194,7 +183,7 @@ module Feedzirra
       end
     
       multi.perform
-      return responses.size == 1 ? responses.values.first : responses.values
+      responses.size == 1 ? responses.values.first : responses.values
     end
     
     # An abstraction for adding a feed by URL to the passed Curb::multi stack.
@@ -216,9 +205,12 @@ module Feedzirra
         curl.headers["User-Agent"]        = (options[:user_agent] || USER_AGENT)
         curl.headers["If-Modified-Since"] = options[:if_modified_since].httpdate if options.has_key?(:if_modified_since)
         curl.headers["If-None-Match"]     = options[:if_none_match] if options.has_key?(:if_none_match)
-        curl.headers["Accept-encoding"]   = 'gzip, deflate'
+        curl.headers["Accept-encoding"]   = 'gzip, deflate' if options.has_key?(:compress)
         curl.follow_location = true
         curl.userpwd = options[:http_authentication].join(':') if options.has_key?(:http_authentication)
+
+        curl.max_redirects = options[:max_redirects] if options[:max_redirects]
+        curl.timeout = options[:timeout] if options[:timeout]
         
         curl.on_success do |c|
           add_url_to_multi(multi, url_queue.shift, url_queue, responses, options) unless url_queue.empty?
@@ -226,12 +218,16 @@ module Feedzirra
           klass = determine_feed_parser_for_xml(xml)
           
           if klass
-            feed = klass.parse(xml)
-            feed.feed_url = c.last_effective_url
-            feed.etag = etag_from_header(c.header_str)
-            feed.last_modified = last_modified_from_header(c.header_str)
-            responses[url] = feed
-            options[:on_success].call(url, feed) if options.has_key?(:on_success)
+            begin
+              feed = klass.parse(xml)
+              feed.feed_url = c.last_effective_url
+              feed.etag = etag_from_header(c.header_str)
+              feed.last_modified = last_modified_from_header(c.header_str)
+              responses[url] = feed
+              options[:on_success].call(url, feed) if options.has_key?(:on_success)
+            rescue Exception => e
+              options[:on_failure].call(url, c.response_code, c.header_str, c.body_str) if options.has_key?(:on_failure)
+            end
           else
             # puts "Error determining parser for #{url} - #{c.last_effective_url}"
             # raise NoParserAvailable.new("no valid parser for content.") (this would unfirtunately fail the whole 'multi', so it's not really useable)
@@ -270,15 +266,22 @@ module Feedzirra
         curl.userpwd = options[:http_authentication].join(':') if options.has_key?(:http_authentication)
         curl.follow_location = true
 
+        curl.max_redirects = options[:max_redirects] if options[:max_redirects]
+        curl.timeout = options[:timeout] if options[:timeout]
+
         curl.on_success do |c|
-          add_feed_to_multi(multi, feed_queue.shift, feed_queue, responses, options) unless feed_queue.empty?
-          updated_feed = Feed.parse(c.body_str)
-          updated_feed.feed_url = c.last_effective_url
-          updated_feed.etag = etag_from_header(c.header_str)
-          updated_feed.last_modified = last_modified_from_header(c.header_str)
-          feed.update_from_feed(updated_feed)
-          responses[feed.feed_url] = feed
-          options[:on_success].call(feed) if options.has_key?(:on_success)
+          begin
+            add_feed_to_multi(multi, feed_queue.shift, feed_queue, responses, options) unless feed_queue.empty?
+            updated_feed = Feed.parse(c.body_str)
+            updated_feed.feed_url = c.last_effective_url
+            updated_feed.etag = etag_from_header(c.header_str)
+            updated_feed.last_modified = last_modified_from_header(c.header_str)
+            feed.update_from_feed(updated_feed)
+            responses[feed.feed_url] = feed
+            options[:on_success].call(feed) if options.has_key?(:on_success)
+          rescue Exception => e
+            options[:on_failure].call(feed, c.response_code, c.header_str, c.body_str) if options.has_key?(:on_failure)
+          end
         end
 
         curl.on_failure do |c|

data/lib/feedzirra/parser/atom.rb

@@ -0,0 +1,47 @@
+module Feedzirra
+  
+  module Parser
+    # == Summary
+    # Parser for dealing with Atom feeds.
+    #
+    # == Attributes
+    # * prev_page
+    # * next_page
+    # * lat_page
+    # * title
+    # * subtitle
+    # * updated
+    # * feed_url
+    # * url
+    # * related
+    # * entries
+    class Atom
+      include SAXMachine
+      include FeedUtilities
+      element :"atom:link", :as => :prev_page, :value => :href, :with => {:rel => 'prev'}
+      element :"atom:link", :as => :next_page, :value => :href, :with => {:rel => 'next'}
+      element :"atom:link", :as => :last_page, :value => :href, :with => {:rel => 'last'}
+      element :title
+      element :subtitle
+      element :updated
+      element :link, :as => :url, :value => :href, :with => {:type => "text/html"}
+      element :link, :as => :feed_url, :value => :href, :with => {:type => "application/atom+xml"}
+      elements :link, :as => :related, :value => :href, :with => {:rel => "related"}
+      elements :link, :as => :links, :value => :href
+      elements :entry, :as => :entries, :class => AtomEntry
+
+      def self.able_to_parse?(xml) #:nodoc:
+        xml =~ /(Atom)|(#{Regexp.escape("http://purl.org/atom")})/
+      end
+      
+      def url
+        @url || links.last
+      end
+      
+      def feed_url
+        @feed_url || links.first
+      end
+    end
+  end
+  
+end

data/lib/feedzirra/parser/atom_entry.rb

@@ -0,0 +1,51 @@
+module Feedzirra
+  
+  module Parser
+    # == Summary
+    # Parser for dealing with Atom feed entries.
+    #
+    # == Attributes
+    # * title
+    # * url
+    # * related
+    # * author
+    # * content
+    # * summary
+    # * published
+    # * categories
+    # * media_content
+    # * media_description
+    # * media_thumbnail
+    # * enclosure
+    class AtomEntry
+      include SAXMachine
+      include FeedEntryUtilities
+      element :title
+      element :link, :as => :url, :value => :href, :with => {:rel => "alternate"}
+      elements :link, :as => :related, :value => :href, :with => {:rel => "related"}
+      element :name, :as => :author
+      element :content
+      element :summary
+      element :published
+      element :id
+      element :created, :as => :published
+      element :issued, :as => :published
+      element :updated
+      element :modified, :as => :updated
+      elements :category, :as => :categories, :value => :term
+
+      element :"media:content", :as => :media_content, :value => :url
+      element :"media:description", :as => :media_description
+      element :"media:thumbnail", :as => :media_thumbnail, :value => :url
+      element :enclosure, :value => :url
+
+      elements :link, :as => :links, :value => :href
+
+      def url
+        @url || links.first
+      end
+    end
+
+  end
+  
+end

data/lib/feedzirra/parser/atom_feed_burner.rb

@@ -0,0 +1,27 @@
+module Feedzirra
+  
+  module Parser
+    # == Summary
+    # Parser for dealing with Feedburner Atom feeds.
+    #
+    # == Attributes
+    # * title
+    # * feed_url
+    # * url
+    # * entries
+    class AtomFeedBurner
+      include SAXMachine
+      include FeedUtilities
+      element :title
+      element :link, :as => :url, :value => :href, :with => {:type => "text/html"}
+      element :link, :as => :feed_url, :value => :href, :with => {:type => "application/atom+xml"}
+      elements :entry, :as => :entries, :class => AtomFeedBurnerEntry
+
+      def self.able_to_parse?(xml) #:nodoc:
+        (xml =~ /Atom/ && xml =~ /feedburner/) || false
+      end
+    end
+
+  end
+  
+end

data/lib/feedzirra/parser/atom_feed_burner_entry.rb

@@ -0,0 +1,35 @@
+module Feedzirra
+  
+  module Parser
+    # == Summary
+    # Parser for dealing with Feedburner Atom feed entries.
+    #
+    # == Attributes
+    # * title
+    # * url
+    # * author
+    # * content
+    # * summary
+    # * published
+    # * categories
+    class AtomFeedBurnerEntry
+      include SAXMachine
+      include FeedEntryUtilities
+      element :title
+      element :name, :as => :author
+      element :link, :as => :url, :value => :href, :with => {:rel => "alternate"}
+      element :"feedburner:origLink", :as => :url
+      element :summary
+      element :content
+      element :published
+      element :id
+      element :issued, :as => :published
+      element :created, :as => :published
+      element :updated
+      element :modified, :as => :updated
+      elements :category, :as => :categories, :value => :term
+    end
+
+  end
+    
+end

data/lib/feedzirra/parser/itunes_rss.rb

@@ -0,0 +1,50 @@
+module Feedzirra
+  
+  module Parser
+    # iTunes is RSS 2.0 + some apple extensions
+    # Source: http://www.apple.com/itunes/whatson/podcasts/specs.html
+    class ITunesRSS
+      include SAXMachine
+      include FeedUtilities
+
+      attr_accessor :feed_url
+
+      # RSS 2.0 elements that need including
+      element :copyright
+      element :description
+      element :language
+      element :managingEditor
+      element :title
+      element :link, :as => :url
+
+      # If author is not present use managingEditor on the channel
+      element :"itunes:author", :as => :itunes_author
+      element :"itunes:block", :as => :itunes_block
+      element :"itunes:image", :value => :href, :as => :itunes_image
+      element :"itunes:explicit", :as => :itunes_explicit
+      element :"itunes:keywords", :as => :itunes_keywords
+      # New URL for the podcast feed
+      element :"itunes:new-feed-url", :as => :itunes_new_feed_url
+      element :"itunes:subtitle", :as => :itunes_subtitle
+      # If summary is not present, use the description tag
+      element :"itunes:summary", :as => :itunes_summary
+
+      # iTunes RSS feeds can have multiple main categories...
+      # ...and multiple sub-categories per category
+      # TODO subcategories not supported correctly - they are at the same level
+      #   as the main categories
+      elements :"itunes:category", :as => :itunes_categories, :value => :text
+
+      elements :"itunes:owner", :as => :itunes_owners, :class => ITunesRSSOwner
+
+      elements :item, :as => :entries, :class => ITunesRSSItem
+
+      def self.able_to_parse?(xml)
+        xml =~ /xmlns:itunes=\"http:\/\/www.itunes.com\/dtds\/podcast-1.0.dtd\"/
+      end
+
+    end
+    
+  end
+  
+end