rubylibre-feedzirra 0.0.14 → 0.0.23

data/lib/feedzirra/parser/rss.rb

@@ -1,28 +1,83 @@
+require File.dirname(__FILE__) + '/mrss_credit'
+require File.dirname(__FILE__) + '/mrss_restriction'
+require File.dirname(__FILE__) + '/mrss_category'
+require File.dirname(__FILE__) + '/mrss_copyright'
+require File.dirname(__FILE__) + '/mrss_hash'
+require File.dirname(__FILE__) + '/mrss_player'
+require File.dirname(__FILE__) + '/mrss_rating'
+require File.dirname(__FILE__) + '/mrss_restriction'
+require File.dirname(__FILE__) + '/mrss_text'
+require File.dirname(__FILE__) + '/mrss_thumbnail'
+
 module Feedzirra
-  
   module Parser
     # == Summary
     # Parser for dealing with RSS feeds.
     #
-    # == Attributes
-    # * title
-    # * feed_url
-    # * url
-    # * entries
     class RSS
       include SAXMachine
       include FeedUtilities
+
+      attr_accessor :feed_url
+
+      # RSS 2.0 required elements
       element :title
       element :link, :as => :url
+      element :description
       elements :item, :as => :entries, :class => RSSEntry
 
-      attr_accessor :feed_url
+      # RSS 2.0 optional elements
+      element :language
+      element :copyright
+      element :managingEditor
+      element :webMaster
+      element :pubDate
+      element :lastBuildDate
+      element :category
+      element :generator
+      element :docs
+      element :cloud
+      element :ttl
+      element :image, :class => RSSImage
+      element :rating
+      element :textInput
+      element :skipHours
+      element :skipDays
+
+      # iTunes
+      element :'itunes:author', :as => :author
+      element :'itunes:block', :as => :itunes_block
+      element :'itunes:image', :as => :image, :value => :href
+      element :'itunes:explicit', :as => :explicit
+      element :'itunes:keywords', :as => :keywords
+      element :'itunes:new-feed-url', :as => :feed_url
+      element :'itunes:name', :as => :owner_name
+      element :'itunes:email', :as => :owner_email
+      element :'itunes:subtitle', :as => :subtitle
+      element :'itunes:summary', :as => :summary
+
+      elements :'itunes:category', :as => :categories, :value => :text
+      # elements :'itunes:category', :as => :itunes_categories,
+      #   :class => ITunesCategory
+
+      # MediaRSS support
+      element :'media:title', :as => :media_title
+      element :'media:keywords', :as => :media_keywords
+      element :'media:description', :as => :media_description
+
+      element :'media:thumbnail', :as => :media_thumbnail, :class => MRSSThumbnail
+      element :'media:rating', :as => :rating, :class => MRSSRating
+      element :'media:category', :as => :media_category, :class => MRSSCategory
+      element :'media:hash', :as => :media_hash, :class => MRSSHash
+      element :'media:player', :as => :media_player, :class => MRSSPlayer
+      elements :'media:credit', :as => :credits, :class => MRSSCredit
+      element :'media:copyright', :as => :copyright, :class => MRSSCopyright
+      element :'media:restriction', :as => :media_restriction, :class => MRSSRestriction
+      element :'media:text', :as => :text, :class => MRSSText
 
       def self.able_to_parse?(xml) #:nodoc:
         xml =~ /\<rss|rdf/
       end
     end
-
   end
-  
 end

data/lib/feedzirra/parser/rss_entry.rb

@@ -1,5 +1,17 @@
+require File.dirname(__FILE__) + '/mrss_content'
+require File.dirname(__FILE__) + '/mrss_credit'
+require File.dirname(__FILE__) + '/mrss_restriction'
+require File.dirname(__FILE__) + '/mrss_group'
+require File.dirname(__FILE__) + '/mrss_category'
+require File.dirname(__FILE__) + '/mrss_copyright'
+require File.dirname(__FILE__) + '/mrss_hash'
+require File.dirname(__FILE__) + '/mrss_player'
+require File.dirname(__FILE__) + '/mrss_rating'
+require File.dirname(__FILE__) + '/mrss_restriction'
+require File.dirname(__FILE__) + '/mrss_text'
+require File.dirname(__FILE__) + '/mrss_thumbnail'
+
 module Feedzirra
-  
   module Parser
     # == Summary
     # Parser for dealing with RDF feed entries.
@@ -15,28 +27,56 @@ module Feedzirra
     class RSSEntry
       include SAXMachine
       include FeedEntryUtilities
+
+      # RSS 2.0 elements
       element :title
       element :link, :as => :url
-
-      element :"dc:creator", :as => :author
-      element :author, :as => :author
-      element :"content:encoded", :as => :content
       element :description, :as => :summary
-
+      element :author
+      elements :category, :as => :categories
+      element :comments
+      element :guid, :as => :id
       element :pubDate, :as => :published
-      element :pubdate, :as => :published
+      element :source
+      element :enclosure, :value => :length, :as => :enclosure_length
+      element :enclosure, :value => :type, :as => :enclosure_type
+      element :enclosure, :value => :url, :as => :enclosure_url
+
+
+      # RDF elements
       element :"dc:date", :as => :published
       element :"dc:Date", :as => :published
       element :"dcterms:created", :as => :published
+      element :issued, :as => :published
+      element :"content:encoded", :as => :content
+      element :"dc:creator", :as => :author
+      element :"dcterms:modified", :as => :updated
 
+      # MediaRSS support, optional elements
+      element :'media:title', :as => :media_title
+      element :'media:keywords', :as => :media_keywords
+      element :'media:description', :as => :media_description
 
-      element :"dcterms:modified", :as => :updated
-      element :issued, :as => :published
-      elements :category, :as => :categories
+      element :'media:thumbnail', :as => :media_thumbnail, :class => MRSSThumbnail
+      element :'media:rating', :as => :rating, :class => MRSSRating
+      element :'media:category', :as => :media_category, :class => MRSSCategory
+      element :'media:hash', :as => :media_hash, :class => MRSSHash
+      element :'media:player', :as => :media_player, :class => MRSSPlayer
+      elements :'media:credit', :as => :credits, :class => MRSSCredit
+      element :'media:copyright', :as => :copyright, :class => MRSSCopyright
+      element :'media:restriction', :as => :media_restriction, :class => MRSSRestriction
+      element :'media:text', :as => :text, :class => MRSSText
+      elements :'media:content', :as => :media_content, :class => MRSSContent
+      elements :'media:group', :as => :media_groups, :class => MRSSGroup
 
-      element :guid, :as => :id
+      # iTunes
+      element :'itunes:author', :as => :author
+      element :'itunes:block', :as => :itunes_block
+      element :'itunes:duration', :as => :duration
+      element :'itunes:explicit', :as => :explicit
+      element :'itunes:keywords', :as => :keywords
+      element :'itunes:subtitle', :as => :subtitle
+      element :'itunes:summary', :as => :summary
     end
-
   end
-  
-end
+end

data/lib/feedzirra/parser/rss_image.rb

@@ -0,0 +1,15 @@
+module Feedzirra
+  module Parser
+    class RSS
+      class RSSImage
+        include SAXMachine
+
+        element :title
+        element :link
+        element :url
+        element :width
+        element :height
+      end
+    end
+  end
+end

data/lib/feedzirra.rb

@@ -11,6 +11,7 @@ require 'active_support/basic_object'
 require 'active_support/core_ext/object'
 require 'active_support/core_ext/time'
 
+
 require 'core_ext/date'
 require 'core_ext/string'
 
@@ -18,17 +19,28 @@ require 'feedzirra/feed_utilities'
 require 'feedzirra/feed_entry_utilities'
 require 'feedzirra/feed'
 
+require 'feedzirra/parser/mrss_content'
+require 'feedzirra/parser/mrss_credit'
+require 'feedzirra/parser/mrss_restriction'
+require 'feedzirra/parser/mrss_group'
+require 'feedzirra/parser/mrss_category'
+require 'feedzirra/parser/mrss_copyright'
+require 'feedzirra/parser/mrss_hash'
+require 'feedzirra/parser/mrss_player'
+require 'feedzirra/parser/mrss_rating'
+require 'feedzirra/parser/mrss_restriction'
+require 'feedzirra/parser/mrss_text'
+require 'feedzirra/parser/mrss_thumbnail'
 require 'feedzirra/parser/rss_entry'
-require 'feedzirra/parser/itunes_rss_owner'
-require 'feedzirra/parser/itunes_rss_item'
+require 'feedzirra/parser/rss_image'
+require 'feedzirra/parser/itunes_category'
 require 'feedzirra/parser/atom_entry'
 require 'feedzirra/parser/atom_feed_burner_entry'
 
 require 'feedzirra/parser/rss'
-require 'feedzirra/parser/itunes_rss'
 require 'feedzirra/parser/atom'
 require 'feedzirra/parser/atom_feed_burner'
 
 module Feedzirra
-  VERSION = "0.0.14"
-end
+  VERSION = "0.0.21"
+end

data/spec/benchmarks/feed_benchmarks.rb

@@ -0,0 +1,98 @@
+# this is some spike code to compare the speed of different methods for performing
+# multiple feed fetches
+require 'rubygems'
+require 'curb'
+require 'activesupport'
+
+require 'net/http'
+require 'uri'
+
+require 'benchmark'
+include Benchmark
+
+GET_COUNT = 1
+urls = ["http://www.pauldix.net"] * GET_COUNT
+
+
+benchmark do |t|
+  t.report("taf2-curb") do
+    multi = Curl::Multi.new
+    urls.each do |url|
+      easy = Curl::Easy.new(url) do |curl|
+        curl.headers["User-Agent"] = "feedzirra"
+    #    curl.headers["If-Modified-Since"] = Time.now.httpdate
+    #    curl.headers["If-None-Match"] = "ziEyTl4q9GH04BR4jgkImd0GvSE"
+        curl.follow_location = true
+        curl.on_success do |c|
+    #      puts c.header_str.inspect
+#          puts c.response_code
+    #      puts c.body_str.slice(0, 500)
+        end
+        curl.on_failure do |c|
+          puts "**** #{c.response_code}"
+        end
+      end
+      multi.add(easy)
+    end
+
+    multi.perform
+  end
+
+  t.report("nethttp") do
+    urls.each do |url|
+      res = Net::HTTP.get(URI.parse(url))
+#      puts res.slice(0, 500)
+    end
+  end
+  
+  require 'rfuzz/session'
+  include RFuzz 
+  t.report("rfuzz") do
+    GET_COUNT.times do
+      http = HttpClient.new("www.pauldix.net", 80)
+      response = http.get("/")
+      if response.http_status != "200" 
+        puts "***** #{response.http_status}"
+      else
+#        puts response.http_status
+  #      puts response.http_body.slice(0, 500)
+      end
+    end
+  end
+  
+  require 'eventmachine'
+  t.report("eventmachine") do
+    counter = GET_COUNT
+    EM.run do
+      GET_COUNT.times do
+        http = EM::Protocols::HttpClient2.connect("www.pauldix.net", 80)
+        request = http.get("/")
+        request.callback do
+#          puts request.status
+#          puts request.content.slice(0, 500)
+          counter -= 1
+          EM.stop if counter == 0
+        end
+      end
+    end
+  end
+  
+  
+  require 'curl-multi'
+  t.report("curl multi") do
+    multi = Curl::Multi.new
+    urls.each do |url|
+      on_failure = lambda do |ex|
+        puts "****** Failed to retrieve #{url}"
+      end
+
+      on_success = lambda do |body|
+#        puts "got #{url}"
+#        puts body.slice(0, 500)
+      end
+      multi.get(url, on_success, on_failure)
+    end
+
+    multi.select([], []) while multi.size > 0
+  end
+end

data/spec/benchmarks/feedzirra_benchmarks.rb

@@ -0,0 +1,40 @@
+require File.dirname(__FILE__) + '/../../lib/feedzirra.rb'
+require 'rfeedparser'
+require 'feed-normalizer'
+require 'open-uri'
+
+require 'benchmark'
+include Benchmark
+
+iterations = 10
+urls = File.readlines(File.dirname(__FILE__) + "/../sample_feeds/successful_feed_urls.txt").slice(0, 20)
+puts "benchmarks on #{urls.size} feeds"
+puts "************************************"
+benchmark do |t|    
+  t.report("feedzirra") do
+    iterations.times do
+      Feedzirra::Feed.fetch_and_parse(urls, :on_success => lambda { |url, feed| $stdout.print '.'; $stdout.flush })
+    end
+  end
+
+  t.report("rfeedparser") do
+    iterations.times do
+      urls.each do |url|
+        feed = FeedParser.parse(url)
+        $stdout.print '.'
+        $stdout.flush
+      end
+    end
+  end
+
+  t.report("feed-normalizer") do
+    iterations.times do
+      urls.each do |url|
+        # have to use the :force option to make feed-normalizer parse an atom feed
+        feed = FeedNormalizer::FeedNormalizer.parse(open(url), :force_parser => FeedNormalizer::SimpleRssParser)
+        $stdout.print '.'
+        $stdout.flush
+      end
+    end
+  end
+end

data/spec/benchmarks/fetching_benchmarks.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require File.dirname(__FILE__) + '/../../lib/feedzirra.rb'
+
+require 'open-uri'
+
+require 'benchmark'
+include Benchmark
+
+iterations = 10
+urls = File.readlines(File.dirname(__FILE__) + "/../sample_feeds/successful_feed_urls.txt").slice(0, 20)
+puts "benchmarks on #{urls.size} feeds"
+puts "************************************"
+benchmark do |t|    
+  t.report("feedzirra open uri") do
+    iterations.times do
+      urls.each do |url|
+        Feedzirra::Feed.parse(open(url, "User-Agent" => "feedzirra http://github.com/pauldix/feedzirra/tree/master").read)
+        $stdout.print '.'; $stdout.flush
+      end
+    end
+  end
+
+  t.report("feedzirra fetch and parse") do
+    iterations.times do
+      Feedzirra::Feed.fetch_and_parse(urls, :on_success => lambda { |url, feed| $stdout.print '.'; $stdout.flush })
+    end
+  end  
+end

data/spec/benchmarks/parsing_benchmark.rb

@@ -0,0 +1,30 @@
+require File.dirname(__FILE__) + '/../../lib/feedzirra.rb'
+require 'rfeedparser'
+require 'feed-normalizer'
+
+require 'benchmark'
+include Benchmark
+
+iterations = 50
+xml = File.read(File.dirname(__FILE__) + '/../sample_feeds/PaulDixExplainsNothing.xml')
+
+benchmark do |t|    
+  t.report("feedzirra") do
+    iterations.times do
+      Feedzirra::Feed.parse(xml)
+    end
+  end
+
+  t.report("rfeedparser") do
+    iterations.times do
+      FeedParser.parse(xml)
+    end
+  end
+
+  t.report("feed-normalizer") do
+    iterations.times do
+      # have to use the :force option to make feed-normalizer parse an atom feed
+      FeedNormalizer::FeedNormalizer.parse(xml, :force_parser => FeedNormalizer::SimpleRssParser)
+    end
+  end
+end

data/spec/benchmarks/updating_benchmarks.rb

@@ -0,0 +1,33 @@
+require 'rubygems'
+require File.dirname(__FILE__) + '/../../lib/feedzirra.rb'
+
+require 'benchmark'
+include Benchmark
+
+urls = File.readlines(File.dirname(__FILE__) + "/../sample_feeds/successful_feed_urls.txt")
+puts "benchmarks on #{urls.size} feeds"
+puts "************************************"
+benchmark do |t|
+  feeds = {}
+  t.report("feedzirra fetch and parse") do
+    feeds = Feedzirra::Feed.fetch_and_parse(urls, 
+      :on_success => lambda { |url, feed| $stdout.print '.'; $stdout.flush },
+      :on_failure => lambda {|url, response_code, header, body| puts "#{response_code} ERROR on #{url}"})
+  end
+  
+  # curb caches the dns lookups for 60 seconds. to make things fair we have to wait for the cache to expire
+  puts "sleeping to wait for dns cache to clear"
+  65.times {$stdout.print('.'); sleep(1)}
+  puts "done"
+  
+  updated_feeds = []
+  t.report("feedzirra update") do
+    updated_feeds = Feedzirra::Feed.update(feeds.values.reject {|f| f.class == Fixnum}, 
+      :on_success => lambda {|feed| $stdout.print '.'; $stdout.flush},
+      :on_failure => lambda {|feed, response_code, header, body| puts "#{response_code} ERROR on #{feed.feed_url}"})
+  end
+  
+  updated_feeds.each do |feed|
+    puts feed.feed_url if feed.updated?
+  end
+end