Spectives-feedzirra 0.0.28

data/lib/feedzirra/parser/mrss_player.rb

@@ -0,0 +1,11 @@
+module Feedzirra
+  module Parser
+    class MRSSPlayer
+      include SAXMachine
+      
+      element :'media:player', :value => :url, :as => :url
+      element :'media:player', :value => :width, :as => :width
+      element :'media:player', :value => :height, :as => :height
+    end
+  end
+end

data/lib/feedzirra/parser/mrss_rating.rb

@@ -0,0 +1,10 @@
+module Feedzirra
+  module Parser
+    class MRSSRating
+      include SAXMachine
+      
+      element :'media:rating', :as => :rating
+      element :'media:rating', :value => :scheme, :as => :scheme
+    end
+  end
+end

data/lib/feedzirra/parser/mrss_restriction.rb

@@ -0,0 +1,11 @@
+module Feedzirra
+  module Parser
+    class MRSSRestriction
+      include SAXMachine
+
+      element :'media:restriction', :as => :value
+      element :'media:restriction', :as => :scope, :value => :type
+      element :'media:restriction', :as => :relationship, :value => :relationship
+    end
+  end
+end

data/lib/feedzirra/parser/mrss_text.rb

@@ -0,0 +1,13 @@
+module Feedzirra
+  module Parser
+    class MRSSText
+      include SAXMachine
+      
+      element :'media:text', :as => :type, :value => :type
+      element :'media:text', :as => :lang, :value => :lang
+      element :'media:text', :as => :start, :value => :start
+      element :'media:text', :as => :end, :value => :end
+      element :'media:text', :as => :text
+    end
+  end
+end

data/lib/feedzirra/parser/mrss_thumbnail.rb

@@ -0,0 +1,11 @@
+module Feedzirra
+  module Parser
+    class MRSSThumbnail
+      include SAXMachine
+      
+      element :'media:thumbnail', :as => :url, :value => :url
+      element :'media:thumbnail', :as => :with, :value => :width
+      element :'media:thumbnail', :as => :height, :value => :height
+    end
+  end
+end

data/lib/feedzirra/parser/rss.rb

@@ -0,0 +1,83 @@
+require File.dirname(__FILE__) + '/mrss_credit'
+require File.dirname(__FILE__) + '/mrss_restriction'
+require File.dirname(__FILE__) + '/mrss_category'
+require File.dirname(__FILE__) + '/mrss_copyright'
+require File.dirname(__FILE__) + '/mrss_hash'
+require File.dirname(__FILE__) + '/mrss_player'
+require File.dirname(__FILE__) + '/mrss_rating'
+require File.dirname(__FILE__) + '/mrss_restriction'
+require File.dirname(__FILE__) + '/mrss_text'
+require File.dirname(__FILE__) + '/mrss_thumbnail'
+
+module Feedzirra
+  module Parser
+    # == Summary
+    # Parser for dealing with RSS feeds.
+    #
+    class RSS
+      include SAXMachine
+      include FeedUtilities
+
+      attr_accessor :feed_url
+
+      # RSS 2.0 required elements
+      element :title
+      element :link, :as => :url
+      element :description
+      elements :item, :as => :entries, :class => RSSEntry
+
+      # RSS 2.0 optional elements
+      element :language
+      element :copyright
+      element :managingEditor
+      element :webMaster
+      element :pubDate
+      element :lastBuildDate
+      element :category
+      element :generator
+      element :docs
+      element :cloud
+      element :ttl
+      element :image, :class => RSSImage
+      element :rating
+      element :textInput
+      element :skipHours
+      element :skipDays
+
+      # iTunes
+      element :'itunes:author', :as => :author
+      element :'itunes:block', :as => :itunes_block
+      element :'itunes:image', :as => :image, :value => :href
+      element :'itunes:explicit', :as => :explicit
+      element :'itunes:keywords', :as => :keywords
+      element :'itunes:new-feed-url', :as => :feed_url
+      element :'itunes:name', :as => :owner_name
+      element :'itunes:email', :as => :owner_email
+      element :'itunes:subtitle', :as => :subtitle
+      element :'itunes:summary', :as => :summary
+
+      elements :'itunes:category', :as => :categories, :value => :text
+      # elements :'itunes:category', :as => :itunes_categories,
+      #   :class => ITunesCategory
+
+      # MediaRSS support
+      element :'media:title', :as => :media_title
+      element :'media:keywords', :as => :media_keywords
+      element :'media:description', :as => :media_description
+
+      element :'media:thumbnail', :as => :media_thumbnail, :class => MRSSThumbnail
+      element :'media:rating', :as => :rating, :class => MRSSRating
+      element :'media:category', :as => :media_category, :class => MRSSCategory
+      element :'media:hash', :as => :media_hash, :class => MRSSHash
+      element :'media:player', :as => :media_player, :class => MRSSPlayer
+      elements :'media:credit', :as => :credits, :class => MRSSCredit
+      element :'media:copyright', :as => :copyright, :class => MRSSCopyright
+      element :'media:restriction', :as => :media_restriction, :class => MRSSRestriction
+      element :'media:text', :as => :text, :class => MRSSText
+
+      def self.able_to_parse?(xml) #:nodoc:
+        xml =~ /\<rss|rdf/
+      end
+    end
+  end
+end

data/lib/feedzirra/parser/rss_entry.rb

@@ -0,0 +1,83 @@
+require File.dirname(__FILE__) + '/mrss_content'
+require File.dirname(__FILE__) + '/mrss_credit'
+require File.dirname(__FILE__) + '/mrss_restriction'
+require File.dirname(__FILE__) + '/mrss_group'
+require File.dirname(__FILE__) + '/mrss_category'
+require File.dirname(__FILE__) + '/mrss_copyright'
+require File.dirname(__FILE__) + '/mrss_hash'
+require File.dirname(__FILE__) + '/mrss_player'
+require File.dirname(__FILE__) + '/mrss_rating'
+require File.dirname(__FILE__) + '/mrss_restriction'
+require File.dirname(__FILE__) + '/mrss_text'
+require File.dirname(__FILE__) + '/mrss_thumbnail'
+
+module Feedzirra
+  module Parser
+    # == Summary
+    # Parser for dealing with RDF feed entries.
+    #
+    # == Attributes
+    # * title
+    # * url
+    # * author
+    # * content
+    # * summary
+    # * published
+    # * categories
+    class RSSEntry
+      include SAXMachine
+      include FeedEntryUtilities
+
+      # RSS 2.0 elements
+      element :title
+      element :link, :as => :url
+      element :"feedburner:origLink", :as => :url
+      element :description, :as => :summary
+      element :author
+      elements :category, :as => :categories
+      element :comments
+      element :guid, :as => :id
+      element :pubDate, :as => :published
+      element :source
+      element :enclosure, :value => :length, :as => :enclosure_length
+      element :enclosure, :value => :type, :as => :enclosure_type
+      element :enclosure, :value => :url, :as => :enclosure_url
+
+
+      # RDF elements
+      element :"dc:date", :as => :published
+      element :"dc:Date", :as => :published
+      element :"dcterms:created", :as => :published
+      element :issued, :as => :published
+      element :"content:encoded", :as => :content
+      element :"dc:creator", :as => :author
+      element :"dcterms:modified", :as => :updated
+
+      # MediaRSS support, optional elements
+      element :'media:title', :as => :media_title
+      element :'media:keywords', :as => :media_keywords
+      element :'media:description', :as => :media_description
+
+      element :'media:thumbnail', :as => :media_thumbnail, :class => MRSSThumbnail
+      element :'media:rating', :as => :rating, :class => MRSSRating
+      element :'media:category', :as => :media_category, :class => MRSSCategory
+      element :'media:hash', :as => :media_hash, :class => MRSSHash
+      element :'media:player', :as => :media_player, :class => MRSSPlayer
+      elements :'media:credit', :as => :credits, :class => MRSSCredit
+      element :'media:copyright', :as => :copyright, :class => MRSSCopyright
+      element :'media:restriction', :as => :media_restriction, :class => MRSSRestriction
+      element :'media:text', :as => :text, :class => MRSSText
+      elements :'media:content', :as => :media_content, :class => MRSSContent
+      elements :'media:group', :as => :media_groups, :class => MRSSGroup
+
+      # iTunes
+      element :'itunes:author', :as => :author
+      element :'itunes:block', :as => :itunes_block
+      element :'itunes:duration', :as => :duration
+      element :'itunes:explicit', :as => :explicit
+      element :'itunes:keywords', :as => :keywords
+      element :'itunes:subtitle', :as => :subtitle
+      element :'itunes:summary', :as => :summary
+    end
+  end
+end

data/lib/feedzirra/parser/rss_image.rb

@@ -0,0 +1,15 @@
+module Feedzirra
+  module Parser
+    class RSS
+      class RSSImage
+        include SAXMachine
+
+        element :title
+        element :link
+        element :url
+        element :width
+        element :height
+      end
+    end
+  end
+end

data/lib/feedzirra.rb

@@ -0,0 +1,44 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__)) unless $LOAD_PATH.include?(File.dirname(__FILE__))
+
+gem 'activesupport'
+
+require 'zlib'
+require 'curb'
+require 'sax-machine'
+require 'dryopteris'
+require 'uri'
+require 'active_support/basic_object'
+require 'active_support/core_ext/object'
+require 'active_support/core_ext/time'
+
+
+require 'core_ext/date'
+require 'core_ext/string'
+
+require 'feedzirra/feed_utilities'
+require 'feedzirra/feed_entry_utilities'
+require 'feedzirra/feed'
+
+require 'feedzirra/parser/mrss_content'
+require 'feedzirra/parser/mrss_credit'
+require 'feedzirra/parser/mrss_restriction'
+require 'feedzirra/parser/mrss_group'
+require 'feedzirra/parser/mrss_category'
+require 'feedzirra/parser/mrss_copyright'
+require 'feedzirra/parser/mrss_hash'
+require 'feedzirra/parser/mrss_player'
+require 'feedzirra/parser/mrss_rating'
+require 'feedzirra/parser/mrss_restriction'
+require 'feedzirra/parser/mrss_text'
+require 'feedzirra/parser/mrss_thumbnail'
+require 'feedzirra/parser/rss_entry'
+require 'feedzirra/parser/rss_image'
+require 'feedzirra/parser/itunes_category'
+require 'feedzirra/parser/atom_entry'
+
+require 'feedzirra/parser/rss'
+require 'feedzirra/parser/atom'
+
+module Feedzirra
+  VERSION = "0.0.28"
+end

data/spec/benchmarks/feed_benchmarks.rb

@@ -0,0 +1,98 @@
+# this is some spike code to compare the speed of different methods for performing
+# multiple feed fetches
+require 'rubygems'
+require 'curb'
+require 'activesupport'
+
+require 'net/http'
+require 'uri'
+
+require 'benchmark'
+include Benchmark
+
+GET_COUNT = 1
+urls = ["http://www.pauldix.net"] * GET_COUNT
+
+
+benchmark do |t|
+  t.report("taf2-curb") do
+    multi = Curl::Multi.new
+    urls.each do |url|
+      easy = Curl::Easy.new(url) do |curl|
+        curl.headers["User-Agent"] = "feedzirra"
+    #    curl.headers["If-Modified-Since"] = Time.now.httpdate
+    #    curl.headers["If-None-Match"] = "ziEyTl4q9GH04BR4jgkImd0GvSE"
+        curl.follow_location = true
+        curl.on_success do |c|
+    #      puts c.header_str.inspect
+#          puts c.response_code
+    #      puts c.body_str.slice(0, 500)
+        end
+        curl.on_failure do |c|
+          puts "**** #{c.response_code}"
+        end
+      end
+      multi.add(easy)
+    end
+
+    multi.perform
+  end
+
+  t.report("nethttp") do
+    urls.each do |url|
+      res = Net::HTTP.get(URI.parse(url))
+#      puts res.slice(0, 500)
+    end
+  end
+  
+  require 'rfuzz/session'
+  include RFuzz 
+  t.report("rfuzz") do
+    GET_COUNT.times do
+      http = HttpClient.new("www.pauldix.net", 80)
+      response = http.get("/")
+      if response.http_status != "200" 
+        puts "***** #{response.http_status}"
+      else
+#        puts response.http_status
+  #      puts response.http_body.slice(0, 500)
+      end
+    end
+  end
+  
+  require 'eventmachine'
+  t.report("eventmachine") do
+    counter = GET_COUNT
+    EM.run do
+      GET_COUNT.times do
+        http = EM::Protocols::HttpClient2.connect("www.pauldix.net", 80)
+        request = http.get("/")
+        request.callback do
+#          puts request.status
+#          puts request.content.slice(0, 500)
+          counter -= 1
+          EM.stop if counter == 0
+        end
+      end
+    end
+  end
+  
+  
+  require 'curl-multi'
+  t.report("curl multi") do
+    multi = Curl::Multi.new
+    urls.each do |url|
+      on_failure = lambda do |ex|
+        puts "****** Failed to retrieve #{url}"
+      end
+
+      on_success = lambda do |body|
+#        puts "got #{url}"
+#        puts body.slice(0, 500)
+      end
+      multi.get(url, on_success, on_failure)
+    end
+
+    multi.select([], []) while multi.size > 0
+  end
+end

data/spec/benchmarks/feedzirra_benchmarks.rb

@@ -0,0 +1,40 @@
+require File.dirname(__FILE__) + '/../../lib/feedzirra.rb'
+require 'rfeedparser'
+require 'feed-normalizer'
+require 'open-uri'
+
+require 'benchmark'
+include Benchmark
+
+iterations = 10
+urls = File.readlines(File.dirname(__FILE__) + "/../sample_feeds/successful_feed_urls.txt").slice(0, 20)
+puts "benchmarks on #{urls.size} feeds"
+puts "************************************"
+benchmark do |t|    
+  t.report("feedzirra") do
+    iterations.times do
+      Feedzirra::Feed.fetch_and_parse(urls, :on_success => lambda { |url, feed| $stdout.print '.'; $stdout.flush })
+    end
+  end
+
+  t.report("rfeedparser") do
+    iterations.times do
+      urls.each do |url|
+        feed = FeedParser.parse(url)
+        $stdout.print '.'
+        $stdout.flush
+      end
+    end
+  end
+
+  t.report("feed-normalizer") do
+    iterations.times do
+      urls.each do |url|
+        # have to use the :force option to make feed-normalizer parse an atom feed
+        feed = FeedNormalizer::FeedNormalizer.parse(open(url), :force_parser => FeedNormalizer::SimpleRssParser)
+        $stdout.print '.'
+        $stdout.flush
+      end
+    end
+  end
+end

data/spec/benchmarks/fetching_benchmarks.rb

@@ -0,0 +1,28 @@
+require 'rubygems'
+require File.dirname(__FILE__) + '/../../lib/feedzirra.rb'
+
+require 'open-uri'
+
+require 'benchmark'
+include Benchmark
+
+iterations = 10
+urls = File.readlines(File.dirname(__FILE__) + "/../sample_feeds/successful_feed_urls.txt").slice(0, 20)
+puts "benchmarks on #{urls.size} feeds"
+puts "************************************"
+benchmark do |t|    
+  t.report("feedzirra open uri") do
+    iterations.times do
+      urls.each do |url|
+        Feedzirra::Feed.parse(open(url, "User-Agent" => "feedzirra http://github.com/pauldix/feedzirra/tree/master").read)
+        $stdout.print '.'; $stdout.flush
+      end
+    end
+  end
+
+  t.report("feedzirra fetch and parse") do
+    iterations.times do
+      Feedzirra::Feed.fetch_and_parse(urls, :on_success => lambda { |url, feed| $stdout.print '.'; $stdout.flush })
+    end
+  end  
+end

data/spec/benchmarks/parsing_benchmark.rb

@@ -0,0 +1,30 @@
+require File.dirname(__FILE__) + '/../../lib/feedzirra.rb'
+require 'rfeedparser'
+require 'feed-normalizer'
+
+require 'benchmark'
+include Benchmark
+
+iterations = 50
+xml = File.read(File.dirname(__FILE__) + '/../sample_feeds/PaulDixExplainsNothing.xml')
+
+benchmark do |t|    
+  t.report("feedzirra") do
+    iterations.times do
+      Feedzirra::Feed.parse(xml)
+    end
+  end
+
+  t.report("rfeedparser") do
+    iterations.times do
+      FeedParser.parse(xml)
+    end
+  end
+
+  t.report("feed-normalizer") do
+    iterations.times do
+      # have to use the :force option to make feed-normalizer parse an atom feed
+      FeedNormalizer::FeedNormalizer.parse(xml, :force_parser => FeedNormalizer::SimpleRssParser)
+    end
+  end
+end

data/spec/benchmarks/updating_benchmarks.rb

@@ -0,0 +1,33 @@
+require 'rubygems'
+require File.dirname(__FILE__) + '/../../lib/feedzirra.rb'
+
+require 'benchmark'
+include Benchmark
+
+urls = File.readlines(File.dirname(__FILE__) + "/../sample_feeds/successful_feed_urls.txt")
+puts "benchmarks on #{urls.size} feeds"
+puts "************************************"
+benchmark do |t|
+  feeds = {}
+  t.report("feedzirra fetch and parse") do
+    feeds = Feedzirra::Feed.fetch_and_parse(urls, 
+      :on_success => lambda { |url, feed| $stdout.print '.'; $stdout.flush },
+      :on_failure => lambda {|url, response_code, header, body| puts "#{response_code} ERROR on #{url}"})
+  end
+  
+  # curb caches the dns lookups for 60 seconds. to make things fair we have to wait for the cache to expire
+  puts "sleeping to wait for dns cache to clear"
+  65.times {$stdout.print('.'); sleep(1)}
+  puts "done"
+  
+  updated_feeds = []
+  t.report("feedzirra update") do
+    updated_feeds = Feedzirra::Feed.update(feeds.values.reject {|f| f.class == Fixnum}, 
+      :on_success => lambda {|feed| $stdout.print '.'; $stdout.flush},
+      :on_failure => lambda {|feed, response_code, header, body| puts "#{response_code} ERROR on #{feed.feed_url}"})
+  end
+  
+  updated_feeds.each do |feed|
+    puts feed.feed_url if feed.updated?
+  end
+end

data/spec/feedzirra/feed_entry_utilities_spec.rb

@@ -0,0 +1,52 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Feedzirra::FeedUtilities do
+  before(:each) do
+    @klass = Class.new do
+      include Feedzirra::FeedEntryUtilities
+    end
+  end
+  
+  describe "handling dates" do
+    it "should parse an ISO 8601 formatted datetime into Time" do
+      time = @klass.new.parse_datetime("2008-02-20T8:05:00-010:00")
+      time.class.should == Time
+      time.to_s.should == "Wed Feb 20 18:05:00 UTC 2008"
+    end
+  end
+  
+  describe "sanitizing" do
+    before(:each) do
+      @feed = Feedzirra::Feed.parse(sample_atom_feed)
+      @entry = @feed.entries.first
+    end
+    
+    it "should provide a sanitized title" do
+      new_title = "<script>" + @entry.title
+      @entry.title = new_title
+      @entry.title.sanitize.should == Dryopteris.sanitize(new_title)
+    end
+    
+    it "should sanitize content in place" do
+      new_content = "<script>" + @entry.content
+      @entry.content = new_content.dup
+      @entry.content.sanitize!.should == Dryopteris.sanitize(new_content)
+      @entry.content.should == Dryopteris.sanitize(new_content)
+    end
+    
+    it "should sanitize things in place" do
+      @entry.title   += "<script>"
+      @entry.author  += "<script>"
+      @entry.content += "<script>"
+
+      cleaned_title   = Dryopteris.sanitize(@entry.title)
+      cleaned_author  = Dryopteris.sanitize(@entry.author)
+      cleaned_content = Dryopteris.sanitize(@entry.content)
+      
+      @entry.sanitize!
+      @entry.title.should   == cleaned_title
+      @entry.author.should  == cleaned_author
+      @entry.content.should == cleaned_content
+    end
+  end
+end