feedzirra 0.0.24 → 0.0.30

data/lib/feedzirra/feed_entry_utilities.rb

@@ -1,34 +1,35 @@
 module Feedzirra
   module FeedEntryUtilities    
+
     def published
-      @published || @updated
+      @published ||= @updated
     end
     
     def parse_datetime(string)
       begin
         DateTime.parse(string).feed_utils_to_gm_time
       rescue
-        puts "DATE CAN'T BE PARSED: #{string}"
+        puts "DATE CAN'T BE PARSED: [#{string}]"
         nil
       end
     end
     
     ##
     # Returns the id of the entry or its url if not id is present, as some formats don't support it
-    def id
-      @entry_id || @url
+    def id  
+      @entry_id ||= @url
     end
-    
+          
     ##
-    # Writter for published. By default, we keep the "oldest" publish time found.
-    def published=(val)
+    # Writer for published. By default, we keep the "oldest" publish time found.
+    def published=(val) 
       parsed = parse_datetime(val)
-      @published = parsed if !@published || parsed < @published
+      @published = parsed if !@published || parsed < @published     
     end
-    
+        
     ##
-    # Writter for udapted. By default, we keep the most recenet update time found.
-    def updated=(val)
+    # Writer for updated. By default, we keep the most recent update time found.
+    def updated=(val) 
       parsed = parse_datetime(val)
       @updated = parsed if !@updated || parsed > @updated
     end

data/lib/feedzirra/parser.rb

@@ -0,0 +1,15 @@
+module Feedzirra
+  module Parser
+    autoload :RSS,      'feedzirra/parser/rss'
+    autoload :RSSEntry, 'feedzirra/parser/rss_entry'
+
+    autoload :ITunesRSS,      'feedzirra/parser/itunes_rss'
+    autoload :ITunesRSSItem,  'feedzirra/parser/itunes_rss_item'
+    autoload :ITunesRSSOwner, 'feedzirra/parser/itunes_rss_owner'
+
+    autoload :Atom,                'feedzirra/parser/atom'
+    autoload :AtomEntry,           'feedzirra/parser/atom_entry'
+    autoload :AtomFeedBurner,      'feedzirra/parser/atom_feed_burner'
+    autoload :AtomFeedBurnerEntry, 'feedzirra/parser/atom_feed_burner_entry'
+  end
+end

data/lib/feedzirra/parser/atom.rb

@@ -1,35 +1,29 @@
 module Feedzirra
-  
+
   module Parser
-    # == Summary
     # Parser for dealing with Atom feeds.
-    #
-    # == Attributes
-    # * title
-    # * feed_url
-    # * url
-    # * entries
     class Atom
       include SAXMachine
       include FeedUtilities
       element :title
+      element :subtitle, :as => :description
       element :link, :as => :url, :value => :href, :with => {:type => "text/html"}
       element :link, :as => :feed_url, :value => :href, :with => {:type => "application/atom+xml"}
       elements :link, :as => :links, :value => :href
       elements :entry, :as => :entries, :class => AtomEntry
 
       def self.able_to_parse?(xml) #:nodoc:
-        xml =~ /(Atom)|(#{Regexp.escape("http://purl.org/atom")})/
+        /\<feed[^\>]+xmlns=[\"|\'](http:\/\/www\.w3\.org\/2005\/Atom|http:\/\/purl\.org\/atom\/ns\#)[\"|\'][^\>]*\>/ =~ xml
       end
-      
+
       def url
         @url || links.last
       end
-      
+
       def feed_url
-        @feed_url || links.first
+        @feed_url ||= links.first
       end
     end
   end
-  
+
 end

data/lib/feedzirra/parser/atom_entry.rb

@@ -1,17 +1,7 @@
 module Feedzirra
-  
+
   module Parser
-    # == Summary
     # Parser for dealing with Atom feed entries.
-    #
-    # == Attributes
-    # * title
-    # * url
-    # * author
-    # * content
-    # * summary
-    # * published
-    # * categories
     class AtomEntry
       include SAXMachine
       include FeedEntryUtilities
@@ -28,12 +18,12 @@ module Feedzirra
       element :modified, :as => :updated
       elements :category, :as => :categories, :value => :term
       elements :link, :as => :links, :value => :href
-      
+
       def url
-        @url || links.first
+        @url ||= links.first
       end
     end
 
   end
-  
+
 end

data/lib/feedzirra/parser/atom_feed_burner.rb

@@ -1,27 +1,21 @@
 module Feedzirra
-  
+
   module Parser
-    # == Summary
     # Parser for dealing with Feedburner Atom feeds.
-    #
-    # == Attributes
-    # * title
-    # * feed_url
-    # * url
-    # * entries
     class AtomFeedBurner
       include SAXMachine
       include FeedUtilities
       element :title
+      element :subtitle, :as => :description
       element :link, :as => :url, :value => :href, :with => {:type => "text/html"}
       element :link, :as => :feed_url, :value => :href, :with => {:type => "application/atom+xml"}
       elements :entry, :as => :entries, :class => AtomFeedBurnerEntry
 
       def self.able_to_parse?(xml) #:nodoc:
-        (xml =~ /Atom/ && xml =~ /feedburner/) || false
+        ((/Atom/ =~ xml) && (/feedburner/ =~ xml)) || false
       end
     end
 
   end
-  
+
 end

data/lib/feedzirra/parser/atom_feed_burner_entry.rb

@@ -1,17 +1,7 @@
 module Feedzirra
-  
+
   module Parser
-    # == Summary
     # Parser for dealing with Feedburner Atom feed entries.
-    #
-    # == Attributes
-    # * title
-    # * url
-    # * author
-    # * content
-    # * summary
-    # * published
-    # * categories
     class AtomFeedBurnerEntry
       include SAXMachine
       include FeedEntryUtilities
@@ -28,8 +18,13 @@ module Feedzirra
       element :updated
       element :modified, :as => :updated
       elements :category, :as => :categories, :value => :term
-    end
+      elements :link, :as => :links, :value => :href
+
+      def url
+        @url ||= links.first
+      end
 
+    end
   end
-    
+
 end

data/lib/feedzirra/parser/itunes_rss.rb

@@ -1,5 +1,5 @@
 module Feedzirra
-  
+
   module Parser
     # iTunes is RSS 2.0 + some apple extensions
     # Source: http://www.apple.com/itunes/whatson/podcasts/specs.html
@@ -40,11 +40,11 @@ module Feedzirra
       elements :item, :as => :entries, :class => ITunesRSSItem
 
       def self.able_to_parse?(xml)
-        xml =~ /xmlns:itunes=\"http:\/\/www.itunes.com\/dtds\/podcast-1.0.dtd\"/
+        /xmlns:itunes=\"http:\/\/www.itunes.com\/dtds\/podcast-1.0.dtd\"/i =~ xml
       end
 
     end
-    
+
   end
-  
+
 end

data/lib/feedzirra/parser/itunes_rss_item.rb

@@ -5,7 +5,7 @@ module Feedzirra
     # Source: http://www.apple.com/itunes/whatson/podcasts/specs.html
     class ITunesRSSItem
       include SAXMachine
-      include FeedUtilities
+      include FeedEntryUtilities
       element :author
       element :guid
       element :title

data/lib/feedzirra/parser/rss.rb

@@ -1,28 +1,22 @@
 module Feedzirra
-  
+
   module Parser
-    # == Summary
     # Parser for dealing with RSS feeds.
-    #
-    # == Attributes
-    # * title
-    # * feed_url
-    # * url
-    # * entries
     class RSS
       include SAXMachine
       include FeedUtilities
       element :title
+      element :description
       element :link, :as => :url
       elements :item, :as => :entries, :class => RSSEntry
 
       attr_accessor :feed_url
 
       def self.able_to_parse?(xml) #:nodoc:
-        xml =~ /\<rss|\<rdf/
+        /\<rss|\<rdf/ =~ xml
       end
     end
 
   end
-  
+
 end

data/lib/feedzirra/parser/rss_entry.rb

@@ -1,17 +1,7 @@
 module Feedzirra
-  
+
   module Parser
-    # == Summary
     # Parser for dealing with RDF feed entries.
-    #
-    # == Attributes
-    # * title
-    # * url
-    # * author
-    # * content
-    # * summary
-    # * published
-    # * categories
     class RSSEntry
       include SAXMachine
       include FeedEntryUtilities
@@ -38,5 +28,5 @@ module Feedzirra
     end
 
   end
-  
+
 end

data/lib/feedzirra/version.rb

@@ -0,0 +1,3 @@
+module Feedzirra
+  VERSION = "0.0.30"
+end

data/spec/benchmarks/feed_benchmarks.rb

@@ -0,0 +1,98 @@
+# this is some spike code to compare the speed of different methods for performing
+# multiple feed fetches
+require 'rubygems'
+require 'curb'
+require 'active_support'
+
+require 'net/http'
+require 'uri'
+
+require 'benchmark'
+include Benchmark
+
+GET_COUNT = 1
+urls = ["http://www.pauldix.net"] * GET_COUNT
+
+
+benchmark do |t|
+  t.report("taf2-curb") do
+    multi = Curl::Multi.new
+    urls.each do |url|
+      easy = Curl::Easy.new(url) do |curl|
+        curl.headers["User-Agent"] = "feedzirra"
+    #    curl.headers["If-Modified-Since"] = Time.now.httpdate
+    #    curl.headers["If-None-Match"] = "ziEyTl4q9GH04BR4jgkImd0GvSE"
+        curl.follow_location = true
+        curl.on_success do |c|
+    #      puts c.header_str.inspect
+#          puts c.response_code
+    #      puts c.body_str.slice(0, 500)
+        end
+        curl.on_failure do |c|
+          puts "**** #{c.response_code}"
+        end
+      end
+      multi.add(easy)
+    end
+
+    multi.perform
+  end
+
+  t.report("nethttp") do
+    urls.each do |url|
+      res = Net::HTTP.get(URI.parse(url))
+#      puts res.slice(0, 500)
+    end
+  end
+  
+  require 'rfuzz/session'
+  include RFuzz 
+  t.report("rfuzz") do
+    GET_COUNT.times do
+      http = HttpClient.new("www.pauldix.net", 80)
+      response = http.get("/")
+      if response.http_status != "200" 
+        puts "***** #{response.http_status}"
+      else
+#        puts response.http_status
+  #      puts response.http_body.slice(0, 500)
+      end
+    end
+  end
+  
+  require 'eventmachine'
+  t.report("eventmachine") do
+    counter = GET_COUNT
+    EM.run do
+      GET_COUNT.times do
+        http = EM::Protocols::HttpClient2.connect("www.pauldix.net", 80)
+        request = http.get("/")
+        request.callback do
+#          puts request.status
+#          puts request.content.slice(0, 500)
+          counter -= 1
+          EM.stop if counter == 0
+        end
+      end
+    end
+  end
+  
+  
+  require 'curl-multi'
+  t.report("curl multi") do
+    multi = Curl::Multi.new
+    urls.each do |url|
+      on_failure = lambda do |ex|
+        puts "****** Failed to retrieve #{url}"
+      end
+
+      on_success = lambda do |body|
+#        puts "got #{url}"
+#        puts body.slice(0, 500)
+      end
+      multi.get(url, on_success, on_failure)
+    end
+
+    multi.select([], []) while multi.size > 0
+  end
+end

data/spec/benchmarks/feedzirra_benchmarks.rb

@@ -0,0 +1,40 @@
+require File.dirname(__FILE__) + '/../../lib/feedzirra.rb'
+require 'rfeedparser'
+require 'feed-normalizer'
+require 'open-uri'
+
+require 'benchmark'
+include Benchmark
+
+iterations = 10
+urls = File.readlines(File.dirname(__FILE__) + "/../sample_feeds/successful_feed_urls.txt").slice(0, 20)
+puts "benchmarks on #{urls.size} feeds"
+puts "************************************"
+benchmark do |t|    
+  t.report("feedzirra") do
+    iterations.times do
+      Feedzirra::Feed.fetch_and_parse(urls, :on_success => lambda { |url, feed| $stdout.print '.'; $stdout.flush })
+    end
+  end
+
+  t.report("rfeedparser") do
+    iterations.times do
+      urls.each do |url|
+        feed = FeedParser.parse(url)
+        $stdout.print '.'
+        $stdout.flush
+      end
+    end
+  end
+
+  t.report("feed-normalizer") do
+    iterations.times do
+      urls.each do |url|
+        # have to use the :force option to make feed-normalizer parse an atom feed
+        feed = FeedNormalizer::FeedNormalizer.parse(open(url), :force_parser => FeedNormalizer::SimpleRssParser)
+        $stdout.print '.'
+        $stdout.flush
+      end
+    end
+  end
+end