feedzirra 0.4.0 → 0.5.0

data/benchmarks/feed_xml/xkcd.xml

@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="utf-8"?>
+<feed xmlns="http://www.w3.org/2005/Atom" xml:lang="en"><title>xkcd.com</title><link href="http://xkcd.com/" rel="alternate"></link><id>http://xkcd.com/</id><updated>2013-11-29T00:00:00Z</updated><entry><title>Oort Cloud</title><link href="http://xkcd.com/1297/" rel="alternate"></link><updated>2013-11-29T00:00:00Z</updated><id>http://xkcd.com/1297/</id><summary type="html">&lt;img src="http://imgs.xkcd.com/comics/oort_cloud.png" title="... I wanna try.  Hang on, be right back." alt="... I wanna try.  Hang on, be right back." /&gt;</summary></entry><entry><title>Git Commit</title><link href="http://xkcd.com/1296/" rel="alternate"></link><updated>2013-11-27T00:00:00Z</updated><id>http://xkcd.com/1296/</id><summary type="html">&lt;img src="http://imgs.xkcd.com/comics/git_commit.png" title="Merge branch 'asdfasjkfdlas/alkdjf' into sdkjfls-final" alt="Merge branch 'asdfasjkfdlas/alkdjf' into sdkjfls-final" /&gt;</summary></entry><entry><title>New Study</title><link href="http://xkcd.com/1295/" rel="alternate"></link><updated>2013-11-25T00:00:00Z</updated><id>http://xkcd.com/1295/</id><summary type="html">&lt;img src="http://imgs.xkcd.com/comics/new_study.png" title="When the results are published, no one will be sure whether to report on them again." alt="When the results are published, no one will be sure whether to report on them again." /&gt;</summary></entry><entry><title>Telescope Names</title><link href="http://xkcd.com/1294/" rel="alternate"></link><updated>2013-11-22T00:00:00Z</updated><id>http://xkcd.com/1294/</id><summary type="html">&lt;img src="http://imgs.xkcd.com/comics/telescope_names.png" title="The Thirty Meter Telescope will be renamed The Flesh-Searing Eye on the Volcano." alt="The Thirty Meter Telescope will be renamed The Flesh-Searing Eye on the Volcano." /&gt;</summary></entry></feed>

data/benchmarks/fetching_systems.rb

@@ -0,0 +1,23 @@
+require 'benchmark'
+require 'net/http'
+require 'curb'
+
+urls = ['http://www.google.com'] * 100
+
+Benchmark.bm(11) do |b|
+  b.report('Net::HTTP') do
+    urls.each do |url|
+      Net::HTTP.get URI.parse url
+    end
+  end
+
+  b.report('Curl::Easy') do
+    urls.each do |url|
+      Curl::Easy.perform url
+    end
+  end
+
+  b.report('Curl::Multi') do
+    Curl::Multi.get urls
+  end
+end

data/benchmarks/other_libraries.rb

@@ -0,0 +1,73 @@
+require 'benchmark'
+require 'feedzirra'
+require 'simple-rss'
+require 'feed-normalizer'
+require 'feed_me'
+
+iterations = 10
+urls = File.readlines(File.dirname(__FILE__) + '/feed_list.txt')
+files = Dir.glob(File.dirname(__FILE__) + '/feed_xml/*.xml')
+xmls = files.map { |file| File.open(file).read }
+
+# suppress warnings
+$VERBOSE = nil
+
+puts 'Parsing benchmarks'
+
+Benchmark.bm(15) do |b|
+  b.report('feedzirra') do
+    iterations.times do
+      xmls.each { |xml| Feedzirra::Feed.parse xml }
+    end
+  end
+
+  b.report('simple-rss') do
+    iterations.times do
+      xmls.each { |xml| SimpleRSS.parse xml }
+    end
+  end
+
+  b.report('feed-normalizer') do
+    iterations.times do
+      xmls.each { |xml| FeedNormalizer::FeedNormalizer.parse xml }
+    end
+  end
+
+  # incompatible with `ruby-feedparser`, same constant used
+  require 'feed_parser'
+  b.report('feed_parser') do
+    iterations.times do
+      xmls.each { |xml| FeedParser.new(feed_xml: xml).parse }
+    end
+  end
+
+  b.report('feed_me') do
+    iterations.times do
+      xmls.each { |xml| FeedMe.parse xml }
+    end
+  end
+
+  # incompatible with `feed_parser`, same constant used
+  # require 'feedparser'
+  # b.report('ruby-feedparser') do
+  #   iterations.times do
+  #     xmls.each { |xml| FeedParser::Feed::new xml }
+  #   end
+  # end
+end
+
+puts "\nFetch and parse benchmarks"
+
+Benchmark.bm(15) do |b|
+  b.report('feedzirra') do
+    iterations.times { Feedzirra::Feed.fetch_and_parse urls }
+  end
+
+  # incompatible with `ruby-feedparser`, same constant used
+  require 'feed_parser'
+  b.report('feed_parser') do
+    iterations.times do
+      urls.each { |url| FeedParser.new(url: url).parse }
+    end
+  end
+end

data/feedzirra.gemspec

@@ -6,7 +6,7 @@ Gem::Specification.new do |s|
   s.version = Feedzirra::VERSION
   s.license = 'MIT'
 
-  s.authors  = ['Paul Dix', 'Julien Kirch', "Ezekiel Templin"]
+  s.authors  = ['Paul Dix', 'Julien Kirch', 'Ezekiel Templin', 'Jon Allured']
   s.email    = 'feedzirra@googlegroups.com'
   s.homepage = 'http://github.com/pauldix/feedzirra'
 
@@ -19,10 +19,9 @@ Gem::Specification.new do |s|
 
   s.platform = Gem::Platform::RUBY
 
-  s.add_dependency 'nokogiri',          '~> 1.6.0'
-  s.add_dependency 'sax-machine',       '~> 0.2.1'
-  s.add_dependency 'curb',              '~> 0.8.1'
-  s.add_dependency 'loofah',            '~> 1.2.1'
+  s.add_dependency 'sax-machine', '~> 0.2.1'
+  s.add_dependency 'curb',        '~> 0.8.1'
+  s.add_dependency 'loofah',      '~> 1.2.1'
 
-  s.add_development_dependency 'rspec', '~> 2.13.0'
+  s.add_development_dependency 'rspec', '~> 2.14.0'
 end

data/lib/feedzirra.rb

@@ -2,7 +2,6 @@ require 'zlib'
 require 'curb'
 require 'sax-machine'
 require 'loofah'
-require 'uri'
 
 require 'feedzirra/core_ext'
 require 'feedzirra/version'

data/lib/feedzirra/feed_utilities.rb

@@ -62,7 +62,7 @@ module Feedzirra
       latest_entry = self.entries.first
       found_new_entries = []
       feed.entries.each do |entry|
-        break if entry.id == latest_entry.id
+        break if entry.entry_id == latest_entry.entry_id || entry.url == latest_entry.url
         found_new_entries << entry
       end
       found_new_entries

data/lib/feedzirra/parser/atom.rb

@@ -14,7 +14,7 @@ module Feedzirra
       elements :entry, :as => :entries, :class => AtomEntry
 
       def self.able_to_parse?(xml) #:nodoc:
-        /\<feed[^\>]+xmlns=[\"|\'](http:\/\/www\.w3\.org\/2005\/Atom|http:\/\/purl\.org\/atom\/ns\#)[\"|\'][^\>]*\>/ =~ xml
+        /\<feed[^\>]+xmlns\s?=\s?[\"|\'](http:\/\/www\.w3\.org\/2005\/Atom|http:\/\/purl\.org\/atom\/ns\#)[\"|\'][^\>]*\>/ =~ xml
       end
 
       def url

data/lib/feedzirra/parser/itunes_rss.rb

@@ -40,7 +40,7 @@ module Feedzirra
       elements :item, :as => :entries, :class => ITunesRSSItem
 
       def self.able_to_parse?(xml)
-        /xmlns:itunes=\"http:\/\/www.itunes.com\/dtds\/podcast-1.0.dtd\"/i =~ xml
+        /xmlns:itunes\s?=\s?\"http:\/\/www.itunes.com\/dtds\/podcast-1.0.dtd\"/i =~ xml
       end
 
     end

data/lib/feedzirra/parser/itunes_rss_item.rb

@@ -22,6 +22,8 @@ module Feedzirra
       element :"itunes:keywords", :as => :itunes_keywords
       element :"itunes:subtitle", :as => :itunes_subtitle
       element :"itunes:image", :value => :href, :as => :itunes_image
+      element :"itunes:isClosedCaptioned", :as => :itunes_closed_captioned
+      element :"itunes:order", :as => :itunes_order
       # If summary is not present, use the description tag
       element :"itunes:summary", :as => :itunes_summary
       element :enclosure, :value => :length, :as => :enclosure_length

data/lib/feedzirra/version.rb

@@ -1,3 +1,3 @@
 module Feedzirra
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
 end

data/spec/feedzirra/feed_spec.rb

@@ -675,19 +675,14 @@ describe Feedzirra::Feed do
     end
 
     describe "#fetch_and_parse" do
-      it 'should initiate the fetching and parsing using multicurl'
-      it "should pass any request options through to add_url_to_multi"
-      it 'should slice the feeds into groups of thirty for processing'
-      it "should return a feed object if a single feed is passed in"
-      it "should return an return an array of feed objects if multiple feeds are passed in"
+      it "passes options to multicurl" do
+        options = { user_agent: '007' }
 
-      it "should set if modified since as an option if passed" do
-        modified_time = Time.parse_safely("Wed, 28 Jan 2009 04:10:32 GMT")
-        Feedzirra::Feed.should_receive(:add_url_to_multi).with(anything, anything, anything, anything, {:if_modified_since => modified_time})
+        Feedzirra::Feed.should_receive(:add_url_to_multi).
+          with(anything, anything, anything, anything, options)
 
-        @feed = Feedzirra::Feed.fetch_and_parse(sample_feedburner_atom_feed, {:if_modified_since => modified_time})
+        Feedzirra::Feed.fetch_and_parse(sample_rss_feed, options)
       end
-
     end
 
     describe "#decode_content" do
@@ -730,10 +725,14 @@ describe Feedzirra::Feed do
     end
 
     describe "#update" do
-      it 'should perform the updating using multicurl'
-      it "should pass any request options through to add_feed_to_multi"
-      it "should return a feed object if a single feed is passed in"
-      it "should return an return an array of feed objects if multiple feeds are passed in"
+      it "passes options to multicurl" do
+        options = { user_agent: '007' }
+
+        Feedzirra::Feed.should_receive(:add_feed_to_multi).
+          with(anything, anything, anything, anything, options)
+
+        Feedzirra::Feed.update(sample_rss_feed, options)
+      end
     end
   end
 end

data/spec/feedzirra/feed_utilities_spec.rb

@@ -122,9 +122,11 @@ describe Feedzirra::FeedUtilities do
         @old_entry = Feedzirra::Parser::AtomEntry.new
         @old_entry.url = "http://pauldix.net/old.html"
         @old_entry.published = Time.now.to_s
+        @old_entry.entry_id = "entry_id_old"
         @new_entry = Feedzirra::Parser::AtomEntry.new
         @new_entry.url = "http://pauldix.net/new.html"
         @new_entry.published = (Time.now + 10).to_s
+        @new_entry.entry_id = "entry_id_new"
         @feed.entries << @old_entry
         @updated_feed.entries << @new_entry
         @updated_feed.entries << @old_entry
@@ -189,5 +191,47 @@ describe Feedzirra::FeedUtilities do
         @feed.new_entries.size.should_not == 2
       end
     end
+
+    describe 'updating with a feed' do
+      let(:id_one) { '1' }
+      let(:id_two) { '2' }
+
+      let(:url_one) { 'http://example.com/post_one.html' }
+      let(:url_two) { 'http://example.com/post_two.html' }
+
+      let(:entry_one) { double 'Entry One', entry_id: id_one, url: url_one }
+      let(:entry_two) { double 'Entry Two', entry_id: id_two, url: url_two }
+
+      let(:feed_one) { Feedzirra::Parser::Atom.new }
+      let(:feed_two) { double 'Feed Two', entries: [entry_two] }
+
+      before do
+        stub_const("Feedzirra::FeedUtilities::UPDATABLE_ATTRIBUTES", [])
+        feed_one.entries << entry_one
+      end
+
+      it 'finds entries with unique ids and urls' do
+        feed_one.update_from_feed feed_two
+        feed_one.new_entries.should eq [entry_two]
+      end
+
+      context 'when the entries have the same id' do
+        let(:id_two) { id_one }
+
+        it 'does not find a new entry' do
+          feed_one.update_from_feed feed_two
+          feed_one.new_entries.should eq []
+        end
+      end
+
+      context 'when the entries have the same url' do
+        let(:url_two) { url_one }
+
+        it 'does not find a new entry' do
+          feed_one.update_from_feed feed_two
+          feed_one.new_entries.should eq []
+        end
+      end
+    end
   end
 end

data/spec/feedzirra/parser/atom_spec.rb

@@ -13,6 +13,10 @@ describe Feedzirra::Parser::Atom do
     it "should return false for an rss feedburner feed" do
       Feedzirra::Parser::Atom.should_not be_able_to_parse(sample_rss_feed_burner_feed)
     end
+
+    it "should return true for an atom feed that has line breaks in between attributes in the <feed> node" do
+      Feedzirra::Parser::Atom.should be_able_to_parse(sample_atom_feed_line_breaks)
+    end
   end
 
   describe "parsing" do

data/spec/feedzirra/parser/itunes_rss_item_spec.rb

@@ -48,4 +48,12 @@ describe Feedzirra::Parser::ITunesRSSItem do
   it "should parse the image" do
     @item.itunes_image.should == "http://example.com/podcasts/everything/AllAboutEverything.jpg"
   end
+
+  it "should parse the order" do
+    @item.itunes_order.should eq '12'
+  end
+
+  it "should parse the closed captioned flag" do
+    @item.itunes_closed_captioned.should eq 'yes'
+  end
 end

data/spec/feedzirra/parser/itunes_rss_spec.rb

@@ -6,6 +6,10 @@ describe Feedzirra::Parser::ITunesRSS do
       Feedzirra::Parser::ITunesRSS.should be_able_to_parse(sample_itunes_feed)
     end
 
+    it "should return true for an itunes RSS feed with spaces between attribute names, equals sign, and values" do
+      Feedzirra::Parser::ITunesRSS.should be_able_to_parse(sample_itunes_feed_with_spaces)
+    end
+
     it "should return fase for an atom feed" do
       Feedzirra::Parser::ITunesRSS.should_not be_able_to_parse(sample_atom_feed)
     end

data/spec/sample_feeds/AtomFeedWithSpacesAroundEquals.xml

@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+This atom/xml feed is an index to active advisories, watches and warnings 
+issued by the National Weather Service.  This index file is not the complete 
+Common Alerting Protocol (CAP) alert message.  To obtain the complete CAP 
+alert, please follow the links for each entry in this index.  Also note the 
+CAP message uses a style sheet to convey the information in a human readable 
+format.  Please view the source of the CAP message to see the complete data 
+set.  Not all information in the CAP message is contained in this index of 
+active alerts.
+-->
+<feed 
+  xmlns = "http://www.w3.org/2005/Atom" 
+  xmlns:cap = "urn:oasis:names:tc:emergency:cap:1.1" 
+  xmlns:ha = "http://www.alerting.net/namespace/index_1.0"
+  >
+  <!-- TZN = <EST> -->
+  <!-- TZO = <-5> -->
+  <!-- http-date = Wed, 07 Nov 2012 02:17:00 GMT -->
+  <id>http://alerts.weather.gov/cap/wwaatmget.php?x=MEC015&amp;y=0</id>
+  <generator>NWS CAP Server</generator>
+  <updated>2012-11-07T09:17:00-05:00</updated>
+  <author>
+    <name>w-nws.webmaster@noaa.gov</name>
+  </author>
+  <title>Current Watches, Warnings and Advisories for Lincoln (MEC015) Maine Issued by the National Weather Service</title>
+  <link href="http://alerts.weather.gov/cap/wwaatmget.php?x=MEC015&amp;y=0"/>
+  <entry>
+    <id>http://alerts.weather.gov/cap/wwacapget.php?x=ME124CCF70CDD4.WinterWeatherAdvisory.124CCF729F10ME.CARWSWCAR.b113b32cf3dd0946aab63451118d16e7</id>
+    <updated>2012-11-07T04:09:00-05:00</updated>
+    <published>2012-11-07T04:09:00-05:00</published>
+    <author>
+      <name>w-nws.webmaster@noaa.gov</name>
+    </author>
+    <title>Winter Weather Advisory issued November 07 at 4:09AM EST until November 08 at 12:00PM EST by NWS</title>
+    <link href="http://alerts.weather.gov/cap/wwacapget.php?x=ME124CCF70CDD4.WinterWeatherAdvisory.124CCF729F10ME.CARWSWCAR.b113b32cf3dd0946aab63451118d16e7"/>
+    <summary>...WINTER WEATHER ADVISORY IN EFFECT FROM 7 PM THIS EVENING TO NOON EST THURSDAY... THE NATIONAL WEATHER SERVICE IN CARIBOU HAS ISSUED A WINTER WEATHER ADVISORY FOR SNOW AND MIXED PRECIPITATION...WHICH IS IN EFFECT FROM 7 PM THIS EVENING TO NOON EST THURSDAY. * PRECIPITATION TYPE...SNOW...SLEET AND FREEZING RAIN</summary>
+    <cap:event>Winter Weather Advisory</cap:event>
+    <cap:effective>2012-11-07T04:09:00-05:00</cap:effective>
+    <cap:expires>2012-11-07T16:00:00-05:00</cap:expires>
+    <cap:status>Actual</cap:status>
+    <cap:msgType>Alert</cap:msgType>
+    <cap:category>Met</cap:category>
+    <cap:urgency>Expected</cap:urgency>
+    <cap:severity>Minor</cap:severity>
+    <cap:certainty>Likely</cap:certainty>
+    <cap:areaDesc>Central Penobscot; Central Washington; Interior Hancock; Northern Penobscot; Northern Washington; Southeast Aroostook; Southern Penobscot; Southern Piscataquis</cap:areaDesc>
+    <cap:polygon/>
+    <cap:geocode>
+      <valueName>FIPS6</valueName>
+      <value>023003 023009 023019 023021 023029</value>
+      <valueName>UGC</valueName>
+      <value>MEZ005 MEZ006 MEZ011 MEZ015 MEZ016 MEZ017 MEZ031 MEZ032</value>
+    </cap:geocode>
+    <cap:parameter>
+      <valueName>VTEC</valueName>
+      <value>/O.NEW.KCAR.WW.Y.0024.121108T0000Z-121108T1700Z/</value>
+    </cap:parameter>
+  </entry>
+</feed>

data/spec/sample_feeds/ITunesWithSpacesInAttributes.xml

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rss 
+  xmlns:itunes = "http://www.itunes.com/dtds/podcast-1.0.dtd" version="2.0"
+  >
+ 
+<channel>
+<title>All About Everything</title>
+<link>http://www.example.com/podcasts/everything/index.html</link>
+<language>en-us</language>
+<copyright>&#x2117; &amp; &#xA9; 2005 John Doe &amp; Family</copyright>
+<itunes:subtitle>A show about everything</itunes:subtitle>
+<itunes:author>John Doe</itunes:author>
+<itunes:summary>All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our Podcast in the iTunes Music Store</itunes:summary>
+<description>All About Everything is a show about everything. Each week we dive into any subject known to man and talk about it as much as we can. Look for our Podcast in the iTunes Music Store</description>
+<itunes:owner>
+<itunes:name>John Doe</itunes:name>
+<itunes:email>john.doe@example.com</itunes:email>
+</itunes:owner>
+<itunes:image href="http://example.com/podcasts/everything/AllAboutEverything.jpg" />
+<itunes:category text="Technology">
+<itunes:category text="Gadgets"/>
+</itunes:category>
+<itunes:category text="TV &amp; Film"/>
+ 
+<item>
+<title>Shake Shake Shake Your Spices</title>
+<itunes:author>John Doe</itunes:author>
+<itunes:subtitle>A short primer on table spices</itunes:subtitle>
+<itunes:summary>This week we talk about salt and pepper shakers, comparing and contrasting pour rates, construction materials, and overall aesthetics. Come and join the party!</itunes:summary>
+<enclosure url="http://example.com/podcasts/everything/AllAboutEverythingEpisode3.m4a" length="8727310" type="audio/x-m4a" />
+<guid>http://example.com/podcasts/archive/aae20050615.m4a</guid>
+<pubDate>Wed, 15 Jun 2005 19:00:00 GMT</pubDate>
+<itunes:duration>7:04</itunes:duration>
+<itunes:keywords>salt, pepper, shaker, exciting</itunes:keywords>
+</item>
+ 
+<item>
+<title>Socket Wrench Shootout</title>
+<itunes:author>Jane Doe</itunes:author>
+<itunes:subtitle>Comparing socket wrenches is fun!</itunes:subtitle>
+<itunes:summary>This week we talk about metric vs. old english socket wrenches. Which one is better? Do you really need both? Get all of your answers here.</itunes:summary>
+<enclosure url="http://example.com/podcasts/everything/AllAboutEverythingEpisode2.mp3" length="5650889" type="audio/mpeg" />
+<guid>http://example.com/podcasts/archive/aae20050608.mp3</guid>
+<pubDate>Wed, 8 Jun 2005 19:00:00 GMT</pubDate>
+<itunes:duration>4:34</itunes:duration>
+<itunes:keywords>metric, socket, wrenches, tool</itunes:keywords>
+</item>
+ 
+<item>
+<title>Red, Whine, &amp; Blue</title>
+<itunes:author>Various</itunes:author>
+<itunes:subtitle>Red + Blue != Purple</itunes:subtitle>
+<itunes:summary>This week we talk about surviving in a Red state if you are a Blue person. Or vice versa.</itunes:summary>
+<enclosure url="http://example.com/podcasts/everything/AllAboutEverythingEpisode1.mp3" length="4989537" type="audio/mpeg" />
+<guid>http://example.com/podcasts/archive/aae20050601.mp3</guid>
+<pubDate>Wed, 1 Jun 2005 19:00:00 GMT</pubDate>
+<itunes:duration>3:59</itunes:duration>
+<itunes:keywords>politics, red, blue, state</itunes:keywords>
+</item>
+ 
+</channel>
+</rss>

data/spec/sample_feeds/itunes.xml

@@ -31,6 +31,8 @@
 <itunes:duration>7:04</itunes:duration>
 <itunes:keywords>salt, pepper, shaker, exciting</itunes:keywords>
 <itunes:image href="http://example.com/podcasts/everything/AllAboutEverything.jpg" />
+<itunes:order>12</itunes:order>
+<itunes:isClosedCaptioned>yes</itunes:isClosedCaptioned>
 </item>
  
 <item>