feedzirra 0.1.3 → 0.2.0.rc1

data/.gitignore

@@ -0,0 +1,12 @@
+.DS_Store
+.rvmrc
+TODO
+Gemfile.lock
+rdoc/
+doc/
+pkg/
+coverage/
+.rbx
+.bundle
+*.swp
+*.swo

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 1.8.7
+  - 1.9.2
+  - 1.9.3
+  - rbx-18mode
+  - rbx-19mode

data/Gemfile

@@ -0,0 +1,9 @@
+source :rubygems
+
+gemspec
+
+group :development, :test do
+  gem 'rake'
+  gem 'guard-rspec'
+  gem 'growl', :require => false
+end

data/Guardfile

@@ -0,0 +1,6 @@
+guard 'rspec', :rvm => ['1.9.3@feedzirra', 'rbx-head@feedzirra'] do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end
+

data/HISTORY.md

@@ -0,0 +1,19 @@
+# 0.2.0.rc1
+* Remove ActiveSupport dependency
+  * No longer tethered to any version of Rails!
+* Update curb (v0.8.0) and rspec (v2.10.0)
+* Revert [3008ceb](https://github.com/pauldix/feedzirra/commit/3008ceb338df1f4c37a211d0aab8a6ad4f584dbc)
+* Add Travis-CI integration
+* General repository and gem maintenance
+
+# 0.1.3
+* ?
+
+# 0.1.2
+* ?
+
+# 0.1.1
+* make FeedEntries enumerable (patch by Daniel Gregoire)
+
+# 0.1.0
+* lower builder requirement to make it rails-3 friendly

data/{README.rdoc → README.md}

@@ -1,10 +1,10 @@
-== Feedzirra
+# Feedzirra
 
-I'd like feedback on the api and any bugs encountered on feeds in the wild. I've set up a {google group here}[http://groups.google.com/group/feedzirra].
+I'd like feedback on the api and any bugs encountered on feeds in the wild. I've set up a [google group here](http://groups.google.com/group/feedzirra).
 
-=== Description
+## Description
 
-Feedzirra is a feed library that is designed to get and update many feeds as quickly as possible. This includes using libcurl-multi through the taf2-curb[link:http://github.com/taf2/curb/tree/master] gem for faster http gets, and libxml through nokogiri[link:http://github.com/tenderlove/nokogiri/tree/master] and sax-machine[link:http://github.com/pauldix/sax-machine/tree/master] for faster parsing.
+Feedzirra is a feed library that is designed to get and update many feeds as quickly as possible. This includes using libcurl-multi through the [curb](https://github.com/taf2/curb) gem for faster http gets, and libxml through [nokogiri](https://github.com/tenderlove/nokogiri) and [sax-machine](https://github.com/pauldix/sax-machine) for faster parsing.
 
 Once you have fetched feeds using Feedzirra, they can be updated using the feed objects. Feedzirra automatically inserts etag and last-modified information from the http response headers to lower bandwidth usage, eliminate unnecessary parsing, and make things speedier in general.
 
@@ -14,13 +14,13 @@ The fetching and parsing logic have been decoupled so that either of them can be
 
 The final feature of Feedzirra is the ability to define custom parsing classes. In truth, Feedzirra could be used to parse much more than feeds. Microformats, page scraping, and almost anything else are fair game.
 
-=== Speedup date parsing
+## Speedup date parsing
 
-In MRI the date parsing code is written in ruby and is optimized for readability over speed, to speed up this part you can install the {home_run}[https://github.com/jeremyevans/home_run] gem to replace it with an optimized C version.
+In MRI the date parsing code is written in ruby and is optimized for readability over speed, to speed up this part you can install the [home_run](https://github.com/jeremyevans/home_run) gem to replace it with an optimized C version.
 
-=== Usage
+## Usage
 
-{A gist of the following code}[link:http://gist.github.com/57285]
+[A gist of the following code](http://gist.github.com/57285)
 
     require 'feedzirra'
 
@@ -70,24 +70,24 @@ In MRI the date parsing code is written in ruby and is optimized for readability
 
     # defining custom behavior on failure or success. note that a return status of 304 (not updated) will call the on_success handler
     feed = Feedzirra::Feed.fetch_and_parse("http://feeds.feedburner.com/PaulDixExplainsNothing",
-        :on_success => lambda {|url, feed| puts feed.title },
-        :on_failure => lambda {|url, response_code, response_header, response_body| puts response_body })
+        :on_success => lambda [|url, feed| puts feed.title ],
+        :on_failure => lambda [|url, response_code, response_header, response_body| puts response_body ])
     # if a collection was passed into fetch_and_parse, the handlers will be called for each one
 
     # the behavior for the handlers when using Feedzirra::Feed.update is slightly different. The feed passed into on_success will be
     # the updated feed with the standard updated accessors. on failure it will be the original feed object passed into update
 
     # fetching a feed via a proxy (optional)
-    feed = Feedzirra::Feed.fetch_and_parse("http://feeds.feedburner.com/PaulDixExplainsNothing", {:proxy_url => '10.0.0.1', :proxy_port => 3084})
+    feed = Feedzirra::Feed.fetch_and_parse("http://feeds.feedburner.com/PaulDixExplainsNothing", [:proxy_url => '10.0.0.1', :proxy_port => 3084])
 
-=== Extending
+## Extending
 
-==== Adding a feed parsing class
+### Adding a feed parsing class
 
     # Adds a new feed parsing class, this class will be used first
     Feedzirra::Feed.add_feed_class MyFeedClass
 
-==== Adding attributes to all feeds types / all entries types
+### Adding attributes to all feeds types / all entries types
 
     # Add the generator attribute to all feed types
     Feedzirra::Feed.add_common_feed_element('generator')
@@ -96,10 +96,10 @@ In MRI the date parsing code is written in ruby and is optimized for readability
     # Add some GeoRss information
     Feedzirra::Feed.add_common_feed_entry_element('geo:lat', :as => :lat)
     Feedzirra::Feed.fetch_and_parse("http://www.earthpublisher.com/georss.php").entries.each do |e|
-        p "lat: #{e.lat}, long: #{e.long}"
+        p "lat: #[e.lat}, long: #{e.long]"
     end
 
-==== Adding attributes to only one class
+### Adding attributes to only one class
 
 If you want to add attributes for only on class you simply have to declare them in the class
 
@@ -113,29 +113,29 @@ If you want to add attributes for only on class you simply have to declare them
 
     # Fetch a feed containing GeoRss info and print them
     Feedzirra::Feed.fetch_and_parse("http://www.earthpublisher.com/georss.php").entries.each do |e|
-        p "lat: #{e.lat}, long: #{e.long}"
+        p "lat: #[e.lat}, long: #{e.long]"
     end
 
-=== Benchmarks
+## Benchmarks
 
-One of the goals of Feedzirra is speed. This includes not only parsing, but fetching multiple feeds as quickly as possible. I ran a benchmark getting 20 feeds 10 times using Feedzirra, rFeedParser, and FeedNormalizer. For more details the {benchmark code can be found in the project in spec/benchmarks/feedzirra_benchmarks.rb}[http://github.com/pauldix/feedzirra/blob/7fb5634c5c16e9c6ec971767b462c6518cd55f5d/spec/benchmarks/feedzirra_benchmarks.rb]
+One of the goals of Feedzirra is speed. This includes not only parsing, but fetching multiple feeds as quickly as possible. I ran a benchmark getting 20 feeds 10 times using Feedzirra, rFeedParser, and FeedNormalizer. For more details the [benchmark code can be found in the project in spec/benchmarks/feedzirra_benchmarks.rb](https://github.com/pauldix/feedzirra/blob/7fb5634c5c16e9c6ec971767b462c6518cd55f5d/spec/benchmarks/feedzirra_benchmarks.rb)
 
     feedzirra          5.170000   1.290000   6.460000 ( 18.917796)
     rfeedparser      104.260000  12.220000 116.480000 (244.799063)
     feed-normalizer   66.250000   4.010000  70.260000 (191.589862)
 
-The result of that benchmark is a bit sketchy because of the network variability. Running 10 times against the same 20 feeds was meant to smooth some of that out. However, there is also a {benchmark comparing parsing speed in spec/benchmarks/parsing_benchmark.rb}[http://github.com/pauldix/feedzirra/blob/7fb5634c5c16e9c6ec971767b462c6518cd55f5d/spec/benchmarks/parsing_benchmark.rb] on an atom feed.
+The result of that benchmark is a bit sketchy because of the network variability. Running 10 times against the same 20 feeds was meant to smooth some of that out. However, there is also a [benchmark comparing parsing speed in spec/benchmarks/parsing_benchmark.rb](https://github.com/pauldix/feedzirra/blob/7fb5634c5c16e9c6ec971767b462c6518cd55f5d/spec/benchmarks/parsing_benchmark.rb) on an atom feed.
 
     feedzirra        0.500000   0.030000   0.530000 (  0.658744)
     rfeedparser      8.400000   1.110000   9.510000 ( 11.839827)
     feed-normalizer  5.980000   0.160000   6.140000 (  7.576140)
 
-There's also a {benchmark that shows the results of using Feedzirra to perform updates on feeds}[http://github.com/pauldix/feedzirra/blob/45d64319544c61a4c9eb9f7f825c73b9f9030cb3/spec/benchmarks/updating_benchmarks.rb] you've already pulled in. I tested against 179 feeds. The first is the initial pull and the second is an update 65 seconds later. I'm not sure how many of them support etag and last-modified, so performance may be better or worse depending on what feeds you're requesting.
+There's also a [benchmark that shows the results of using Feedzirra to perform updates on feeds](https://github.com/pauldix/feedzirra/blob/45d64319544c61a4c9eb9f7f825c73b9f9030cb3/spec/benchmarks/updating_benchmarks.rb) you've already pulled in. I tested against 179 feeds. The first is the initial pull and the second is an update 65 seconds later. I'm not sure how many of them support etag and last-modified, so performance may be better or worse depending on what feeds you're requesting.
 
     feedzirra fetch and parse  4.010000   0.710000   4.720000 ( 15.110101)
     feedzirra update           0.660000   0.280000   0.940000 (  5.152709)
 
-=== TODO
+## TODO
 
 This thing needs to hammer on many different feeds in the wild. I'm sure there will be bugs. I want to find them and crush them. I didn't bother using the test suite for feedparser. i wanted to start fresh.
 
@@ -149,14 +149,16 @@ Here are some more specific TODOs.
 * Make the feed_spec actually mock stuff out so it doesn't hit the net.
 * Readdress how feeds determine if they can parse a document. Maybe I should use namespaces instead?
 
-=== LICENSE
+## LICENSE
 
 (The MIT License)
  
-Copyright (c) 2009:
- 
-{Paul Dix}[http://pauldix.net]
+Copyright (c) 2009-2012:
  
+- [Paul Dix](http://pauldix.net)
+- [Julien Kirch](http://archiloque.net/)
+- [Ezekiel Templin](http://zeke.templ.in/)
+
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
 'Software'), to deal in the Software without restriction, including

data/Rakefile

@@ -1,51 +1,6 @@
-require 'bundler'
-Bundler.setup
-
-require 'rake'
-require 'rdoc/task'
-require 'rspec'
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
 require 'rspec/core/rake_task'
-
-$LOAD_PATH.unshift File.expand_path('../lib', __FILE__)
-require 'feedzirra/version'
-
-def recent_specs(touched_since)
-  recent_specs = FileList['app/**/*'].map do |path|
-
-    if File.mtime(path) > touched_since
-      spec = File.join('spec', File.dirname(path).split("/")[1..-1].join('/'),
-        "#{File.basename(path, ".*")}_spec.rb")
-      spec if File.exists?(spec)
-    end
-  end.compact
-
-  recent_specs += FileList['spec/**/*_spec.rb'].select do |path|
-    File.mtime(path) > touched_since
-  end
-  recent_specs.uniq
-end
-
-RSpec::Core::RakeTask.new do |t|
-  t.pattern = FileList['spec/**/*_spec.rb']
-end
-
-desc 'Run recent specs'
-RSpec::Core::RakeTask.new("spec:recent") do |t|
-  t.pattern = recent_specs(Time.now - 600) # 10 min.
-end
-
-RSpec::Core::RakeTask.new('spec:rcov') do |t|
-  t.pattern = FileList['spec/**/*_spec.rb']
-  t.rcov = true
-  t.rcov_opts = ['--exclude', 'spec,/usr/lib/ruby,/usr/local,/var/lib,/Library', '--text-report']
-end
-
-RDoc::Task.new do |rd|
-  rd.title    = 'Feedzirra'
-  rd.rdoc_dir = 'rdoc'
-  rd.rdoc_files.include('README.rdoc', 'lib/feedzirra.rb', 'lib/feedzirra/**/*.rb')
-  rd.options = ["--quiet", "--opname", "index.html", "--line-numbers", "--inline-source", '--main', 'README.rdoc']
-end
-
-desc "Run all the tests"
-task :default => :spec
+RSpec::Core::RakeTask.new(:spec)
+task :test => :spec
+task :default => :test

data/feedzirra.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/feedzirra/version', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name    = 'feedzirra'
+  s.version = Feedzirra::VERSION
+
+  s.authors  = ['Paul Dix', 'Julien Kirch', "Ezekiel Templin"]
+  s.date     = Date.today
+  s.email    = 'feedzirra@googlegroups.com'
+  s.homepage = 'http://github.com/pauldix/feedzirra'
+
+  s.summary     = 'A feed fetching and parsing library'
+  s.description = 'A feed fetching and parsing library that treats the internet like Godzilla treats Japan: it dominates and eats all.'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_paths = ['lib']
+  
+  s.platform = Gem::Platform::RUBY
+
+  s.add_dependency 'nokogiri',          '~> 1.5.2'
+  s.add_dependency 'sax-machine',       '~> 0.1.0'
+  s.add_dependency 'curb',              '~> 0.8.0'
+  s.add_dependency 'loofah',            '~> 1.2.1'
+  s.add_dependency 'gorillib',          '~> 0.1.9'
+
+  s.add_development_dependency 'rspec', '~> 2.10.0'
+end

data/lib/feedzirra.rb

@@ -3,11 +3,7 @@ require 'curb'
 require 'sax-machine'
 require 'loofah'
 require 'uri'
-
-require 'active_support/basic_object'
-require 'active_support/core_ext/module'
-require 'active_support/core_ext/object'
-require 'active_support/time'
+require 'gorillib/datetime/parse'
 
 require 'feedzirra/core_ext'
 
@@ -16,4 +12,6 @@ module Feedzirra
   autoload :FeedUtilities,      'feedzirra/feed_utilities'
   autoload :Feed,               'feedzirra/feed'
   autoload :Parser,             'feedzirra/parser'
+  
+  class NoParserAvailable < StandardError; end
 end

data/lib/feedzirra/feed.rb

@@ -1,6 +1,4 @@
-module Feedzirra
-  class NoParserAvailable < StandardError; end
-  
+module Feedzirra  
   class Feed
     USER_AGENT = "feedzirra http://github.com/pauldix/feedzirra/tree/master"
     

data/lib/feedzirra/feed_utilities.rb

@@ -50,12 +50,23 @@ module Feedzirra
     private
 
     def find_new_entries_for(feed)
-      # this algorithm does not optimize based on publication date, but always finds new entries
-      feed.entries.reject {|entry| self.entries.any? {|e| e.url == entry.url} }
+      # this implementation is a hack, which is why it's so ugly.
+      # it's to get around the fact that not all feeds have a published date.
+      # however, they're always ordered with the newest one first.
+      # So we go through the entries just parsed and insert each one as a new entry
+      # until we get to one that has the same url as the the newest for the feed
+      return feed.entries if self.entries.length == 0
+      latest_entry = self.entries.first
+      found_new_entries = []
+      feed.entries.each do |entry|
+        break if entry.url == latest_entry.url
+        found_new_entries << entry
+      end
+      found_new_entries
     end
 
     def existing_entry?(test_entry)
       entries.any? { |entry| entry.url == test_entry.url }
     end
   end
-end
+end

data/lib/feedzirra/version.rb

@@ -1,3 +1,3 @@
 module Feedzirra
-  VERSION = '0.1.3'
+  VERSION = '0.2.0.rc1'
 end

data/spec/feedzirra/feed_entry_utilities_spec.rb

@@ -11,7 +11,7 @@ describe Feedzirra::FeedUtilities do
     it "should parse an ISO 8601 formatted datetime into Time" do
       time = @klass.new.parse_datetime("2008-02-20T8:05:00-010:00")
       time.class.should == Time
-      time.to_s.should == "Wed Feb 20 18:05:00 UTC 2008"
+      time.should == Time.parse_safely("Wed Feb 20 18:05:00 UTC 2008")
     end
   end
   

data/spec/feedzirra/feed_spec.rb

@@ -43,35 +43,35 @@ describe Feedzirra::Feed do
       it "should parse an rdf feed" do
         feed = Feedzirra::Feed.parse(sample_rdf_feed)
         feed.title.should == "HREF Considered Harmful"
-        feed.entries.first.published.to_s.should == "Tue Sep 02 19:50:07 UTC 2008"
+        feed.entries.first.published.should == Time.parse_safely("Tue Sep 02 19:50:07 UTC 2008")
         feed.entries.size.should == 10
       end
 
       it "should parse an rss feed" do
         feed = Feedzirra::Feed.parse(sample_rss_feed)
         feed.title.should == "Tender Lovemaking"
-        feed.entries.first.published.to_s.should == "Thu Dec 04 17:17:49 UTC 2008"
+        feed.entries.first.published.should == Time.parse_safely("Thu Dec 04 17:17:49 UTC 2008")
         feed.entries.size.should == 10
       end
 
       it "should parse an atom feed" do
         feed = Feedzirra::Feed.parse(sample_atom_feed)
         feed.title.should == "Amazon Web Services Blog"
-        feed.entries.first.published.to_s.should == "Fri Jan 16 18:21:00 UTC 2009"
+        feed.entries.first.published.should == Time.parse_safely("Fri Jan 16 18:21:00 UTC 2009")
         feed.entries.size.should == 10
       end
 
       it "should parse an feedburner atom feed" do
         feed = Feedzirra::Feed.parse(sample_feedburner_atom_feed)
         feed.title.should == "Paul Dix Explains Nothing"
-        feed.entries.first.published.to_s.should == "Thu Jan 22 15:50:22 UTC 2009"
+        feed.entries.first.published.should == Time.parse_safely("Thu Jan 22 15:50:22 UTC 2009")
         feed.entries.size.should == 5
       end      
 
       it "should parse an itunes feed as a standard RSS feed" do
         feed = Feedzirra::Feed.parse(sample_itunes_feed)
         feed.title.should == "All About Everything"
-        feed.entries.first.published.should == Time.parse("Wed, 15 Jun 2005 19:00:00 GMT")
+        feed.entries.first.published.should == Time.parse_safely("Wed, 15 Jun 2005 19:00:00 GMT")
         
         # Since the commit 621957879, iTunes feeds will be parsed as standard RSS, so this
         # entry should now not have a method for itunes_author.
@@ -91,7 +91,7 @@ describe Feedzirra::Feed do
     it "should parse an feedburner rss feed" do
       feed = Feedzirra::Feed.parse(sample_rss_feed_burner_feed)
       feed.title.should == "TechCrunch"
-      feed.entries.first.published.to_s.should == "Wed Nov 02 17:25:27 UTC 2011"
+      feed.entries.first.published.should == Time.parse_safely("Wed Nov 02 17:25:27 UTC 2011")
       feed.entries.size.should == 20
     end
   end
@@ -167,7 +167,7 @@ describe Feedzirra::Feed do
     end
 
     it "should return the last modified date from the header if it exists" do
-      Feedzirra::Feed.last_modified_from_header(@header).should == Time.parse("Wed, 28 Jan 2009 04:10:32 GMT")
+      Feedzirra::Feed.last_modified_from_header(@header).should == Time.parse_safely("Wed, 28 Jan 2009 04:10:32 GMT")
     end
 
     it "should return nil if there is no last modified date in the header" do
@@ -204,7 +204,7 @@ describe Feedzirra::Feed do
       end
       
       it "should set if modified since as an option if passed" do
-        Feedzirra::Feed.fetch_raw(@paul_feed[:url], :if_modified_since => Time.parse("Wed, 28 Jan 2009 04:10:32 GMT"))
+        Feedzirra::Feed.fetch_raw(@paul_feed[:url], :if_modified_since => Time.parse_safely("Wed, 28 Jan 2009 04:10:32 GMT"))
         @curl.headers["If-Modified-Since"].should == 'Wed, 28 Jan 2009 04:10:32 GMT'
       end
 
@@ -258,6 +258,7 @@ describe Feedzirra::Feed do
 
     describe "#add_url_to_multi" do
       before(:each) do
+        allow_message_expectations_on_nil
         @multi = Curl::Multi.get([@paul_feed[:url]], {:follow_location => true}, {:pipeline => true})
         @multi.stub!(:add)
         @easy_curl = Curl::Easy.new(@paul_feed[:url])
@@ -276,7 +277,7 @@ describe Feedzirra::Feed do
       end
       
       it "should set if modified since as an option if passed" do
-        Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, :if_modified_since => Time.parse("Jan 25 2009 04:10:32 GMT"))
+        Feedzirra::Feed.add_url_to_multi(@multi, @paul_feed[:url], [], {}, :if_modified_since => Time.parse_safely("Jan 25 2009 04:10:32 GMT"))
         @easy_curl.headers["If-Modified-Since"].should == 'Sun, 25 Jan 2009 04:10:32 GMT'
       end
 
@@ -400,6 +401,7 @@ describe Feedzirra::Feed do
 
     describe "#add_feed_to_multi" do
       before(:each) do
+        allow_message_expectations_on_nil
         @multi = Curl::Multi.get([@paul_feed[:url]], {:follow_location => true}, {:pipeline => true})
         @multi.stub!(:add)
         @easy_curl = Curl::Easy.new(@paul_feed[:url])
@@ -419,7 +421,7 @@ describe Feedzirra::Feed do
       end
 
       it "should set if modified since as an option if passed" do
-        modified_time = Time.parse("Wed, 28 Jan 2009 04:10:32 GMT")
+        modified_time = Time.parse_safely("Wed, 28 Jan 2009 04:10:32 GMT")
         Feedzirra::Feed.add_feed_to_multi(@multi, @feed, [], {}, {:if_modified_since => modified_time})
         modified_time.should be > @feed.last_modified
         
@@ -540,7 +542,7 @@ describe Feedzirra::Feed do
       it "should return an return an array of feed objects if multiple feeds are passed in"
       
       it "should set if modified since as an option if passed" do
-        modified_time = Time.parse("Wed, 28 Jan 2009 04:10:32 GMT")
+        modified_time = Time.parse_safely("Wed, 28 Jan 2009 04:10:32 GMT")
         Feedzirra::Feed.should_receive(:add_url_to_multi).with(anything, anything, anything, anything, {:if_modified_since => modified_time}).any_number_of_times
         
         @feed = Feedzirra::Feed.fetch_and_parse(sample_feedburner_atom_feed, {:if_modified_since => modified_time})