feed_parser 0.3.4 → 0.3.5

data/.travis.yml

@@ -1,3 +1,3 @@
 rvm:
-  - 1.8.7
   - 1.9.3
+  - 2.0.0

data/Gemfile

@@ -1,4 +1,4 @@
-source :rubygems
+source 'http://rubygems.org'
 
 gem 'nokogiri'
 
@@ -6,7 +6,3 @@ group :test do
   gem 'rake'
   gem 'rspec'
 end
-
-platform :jruby do
-  gem 'jruby-openssl'
-end

data/README.md

@@ -6,7 +6,8 @@ Rss and Atom feed parser built on top of Nokogiri. Supports custom sanitizers.
 
 [![Build Status](https://secure.travis-ci.org/arttu/feed_parser.png)](http://travis-ci.org/arttu/feed_parser)
 
-FeedParser gem is tested on Ruby 1.8.7 and 1.9.3.
+FeedParser gem is tested on Ruby 1.9.3 and 2.0.0.
+1.8.7 should work with Nokogiri < 1.6.0.
 
 ## Install
 
@@ -16,17 +17,28 @@ Add to Gemfile
 
 ## Usage
 
-    # the most basic use case
+#### Parse from URL
+
     fp = FeedParser.new(:url => "http://example.com/feed/")
-    # with sanitizer
+    feed = fp.parse
+
+Optionally pass HTTP options, see more from the OpenURI documentation: http://apidock.com/ruby/OpenURI
+
+    fp = FeedParser.new(:url => "http://example.com/feed/", :http => {:ssl_verify_mode => OpenSSL::SSL::VERIFY_NONE})
+
+#### Parse from an XML string
+
+    fp = FeedParser.new(:feed_xml => "<rss>...</rss>")
+    feed = fp.parse
+
+#### Use sanitizer
+
     fp = FeedParser.new(:url => "http://example.com/feed/", :sanitizer => MyBestestSanitizer.new)
     # sanitizing custom field set
     fp = FeedParser.new(:url => "http://example.com/feed/", :sanitizer => MyBestestSanitizer.new, :fields_to_sanitize => [:title, :content])
-    
-    # retrieve the feed xml and parse it
-    feed = fp.parse
-    
-    # using parsed feed in your code
+
+#### Using parsed feed in your code
+
     feed.as_json
     # => {:title => "Feed title", :url => "http://example.com/feed/", :items => [{:guid => , :title => , :author => ...}]}
     
@@ -34,11 +46,7 @@ Add to Gemfile
       pp feed_item
     end
 
-    # you can also pass http options to be used for the connection
-    # for available options, check out the OpenURI documentation: http://apidock.com/ruby/OpenURI
-    fp = FeedParser.new(:url => "http://example.com/feed/", :http => {:ssl_verify_mode => OpenSSL::SSL::VERIFY_NONE})
-
-If the fetched XML is not a valid RSS or an ATOM feed, a FeedParser::UnknownFeedType is raised in FeedParser#parse.
+If the XML is not a valid RSS or an ATOM feed, a FeedParser::UnknownFeedType is raised in FeedParser#parse.
 
 ## Running tests
 

data/Rakefile

@@ -18,18 +18,3 @@ end
 
 desc "Default: Run specs"
 task :default => :spec
-
-namespace :rubies do
-  rvm_rubies_command = "rvm 1.8.7-p302@feed_parser,1.9.3-p194@feed_parser do"
-
-  desc "Update dependencies for all Ruby versions"
-  task :update_dependencies do
-    system("#{rvm_rubies_command} bundle install")
-    system("#{rvm_rubies_command} bundle update")
-  end
-
-  desc "Run tests with Ruby versions 1.8.7 and 1.9.3"
-  task :spec do
-    system("#{rvm_rubies_command} bundle exec rake spec")
-  end
-end

data/feed_parser.gemspec

@@ -1,12 +1,9 @@
 # feed_parser.gemspec
 # -*- encoding: utf-8 -*-
 
-$:.push File.expand_path("../lib", __FILE__)
-require 'feed_parser'
-
 Gem::Specification.new do |s|
   s.name        = 'feed_parser'
-  s.version     = FeedParser::VERSION
+  s.version     = "0.3.5"
   s.authors     = ['Arttu Tervo']
   s.email       = ['arttu.tervo@gmail.com']
   s.homepage    = 'https://github.com/arttu/feed_parser'

data/lib/feed_parser.rb

@@ -3,18 +3,22 @@ require 'nokogiri'
 
 class FeedParser
 
-  VERSION = "0.3.4"
-
   USER_AGENT = "Ruby / FeedParser gem"
 
   class FeedParser::UnknownFeedType < Exception ; end
   class FeedParser::InvalidURI < Exception ; end
 
+  def self.parse(opts)
+    fp = FeedParser.new(opts)
+    fp.parse
+  end
+
   def initialize(opts)
     @url = opts[:url]
     @http_options = {"User-Agent" => FeedParser::USER_AGENT}.merge(opts[:http] || {})
     @@sanitizer = (opts[:sanitizer] || SelfSanitizer.new)
     @@fields_to_sanitize = (opts[:fields_to_sanitize] || [:content])
+    @feed_xml = opts[:feed_xml]
     self
   end
 
@@ -27,7 +31,11 @@ class FeedParser
   end
 
   def parse
-    feed_xml = open_or_follow_redirect(@url)
+    if @feed_xml
+      feed_xml = @feed_xml
+    else
+      feed_xml = open_or_follow_redirect(@url)
+    end
     @feed ||= Feed.new(feed_xml)
     feed_xml.close! if feed_xml.class.to_s == 'Tempfile'
     @feed

data/lib/feed_parser/dsl.rb

@@ -11,6 +11,7 @@ class FeedParser
         :item_guid => "guid",
         :item_link => "link",
         :item_title => "title",
+        :item_published => "pubDate",
         :item_categories => "category",
         :item_author => "creator",
         :item_description => "description",
@@ -26,6 +27,8 @@ class FeedParser
         :item_guid => "id",
         :item_link => "link",
         :item_title => "title",
+        :item_published => "published",
+        :item_updated => "updated",
         :item_categories => "category",
         :item_author => "author/name",
         :item_description => "summary",

data/lib/feed_parser/feed_item.rb

@@ -7,6 +7,7 @@ class FeedParser
     def initialize(item)
       @guid = item.xpath(Dsl[@type][:item_guid]).text
       @title = item.xpath(Dsl[@type][:item_title]).text
+      @published = parse_datetime(item.xpath(Dsl[@type][:item_published]).text)
       @author = item.xpath(Dsl[@type][:item_author]).text
       @description = possible_html_content(item.xpath(Dsl[@type][:item_description]))
       @content = possible_html_content(item.xpath(Dsl[@type][:item_content]))
@@ -27,13 +28,14 @@ class FeedParser
 
     def as_json
       {
-        :guid => guid,
-        :link => link,
-        :title => title,
-        :categories => categories,
-        :author => author,
-        :description => description,
-        :content => content
+        :guid => self.guid,
+        :link => self.link,
+        :title => self.title,
+        :published => self.published,
+        :categories => self.categories,
+        :author => self.author,
+        :description => self.description,
+        :content => self.content
       }
     end
 
@@ -51,6 +53,15 @@ class FeedParser
           element.text
       end
     end
+
+    def parse_datetime(string)
+      begin
+        DateTime.parse(string) unless string.empty?
+      rescue
+        warn "Failed to parse date #{string.inspect}"
+        nil
+      end
+    end
   end
 
   class RssItem < FeedItem
@@ -67,7 +78,12 @@ class FeedParser
       @type = :atom
       super
       @link = item.xpath(Dsl[@type][:item_link]).attribute("href").text.strip
+      @updated = parse_datetime(item.xpath(Dsl[@type][:item_updated]).text)
       @categories = item.xpath(Dsl[@type][:item_categories]).map{|cat| cat.attribute("term").text}
     end
+
+    def published
+      @published ||= @updated
+    end
   end
 end

data/spec/feed_parser_spec.rb

@@ -18,6 +18,19 @@ describe FeedParser do
     opts
   end
 
+  describe ".parse" do
+    it "should instantiate a new FeedParser and return a parsed feed" do
+      feed = FeedParser::Feed.new(feed_xml)
+
+      fp = FeedParser.new(:url => "http://blog.example.com/feed/")
+      fp.should_receive(:parse).and_return(feed)
+
+      FeedParser.should_receive(:new).with(:url => "http://blog.example.com/feed/").and_return(fp)
+
+      FeedParser.parse(:url => "http://blog.example.com/feed/").should == feed
+    end
+  end
+
   describe "#new" do
     it "should forward given http options to the OpenURI" do
       FeedParser.any_instance.should_receive(:open).with("http://blog.example.com/feed/", http_connection_options.merge(:ssl_verify_mode => OpenSSL::SSL::VERIFY_NONE)).and_return(feed_xml)
@@ -73,6 +86,11 @@ describe FeedParser do
         fp.parse
       }.should raise_error(FeedParser::InvalidURI, "Only URIs with http or https protocol are supported")
     end
+
+    it "should parse feeds from the :feed_xml option instead of the :url" do
+      fp = FeedParser.new(:feed_xml => "<rss><channel><item><title>feed_xml test</title><link>http://example.com</link></item></channel></rss>")
+      fp.parse.items.first.title == "feed_xml test"
+    end
   end
 
   describe "::Feed" do
@@ -120,6 +138,12 @@ describe FeedParser do
         end
       end
 
+      it "should set the published date" do
+        feed = FeedParser::Feed.new(feed_xml('nodeta.rss.xml'))
+        item = feed.items.first
+        item.published.should == DateTime.parse("Jul 5, 2009 09:25:32 GMT")
+      end
+
       {
         'nodeta.rss.xml' => {
           :title => "Nodeta",
@@ -129,6 +153,7 @@ describe FeedParser do
                 :guid => "http://blog.nodeta.fi/?p=73",
                 :link => "http://blog.nodeta.fi/2009/01/16/ruby-187-imported/",
                 :title => "Ruby 1.8.7 imported",
+                :published => DateTime.parse("Jan 16, 2009 15:29:52 GMT"),
                 :categories => ["APIdock", "Ruby"],
                 :author => "Otto Hilska",
                 :description => "I just finished importing Ruby 1.8.7 to APIdock. It&#8217;s also the new default version, because usually it is better documented. However, there&#8217;re some incompatibilities between 1.8.6 and 1.8.7, so be sure to check the older documentation when something seems to be wrong.\n",
@@ -144,6 +169,7 @@ describe FeedParser do
                 :guid => "basecamp.00000000.Comment.1234567",
                 :link => "https://awesome.basecamphq.com/unique_item_link",
                 :title => "Comment posted: Re: Howdy how?",
+                :published => DateTime.parse("Nov 9, 2011 20:35:18 GMT"),
                 :categories => [],
                 :author => "Ffuuuuuuu- Le.",
                 :description => "<div>trololooo</div><p>Company: awesome | Project: Awesome project</p>",
@@ -159,6 +185,7 @@ describe FeedParser do
               :guid => "http://scrumalliance.org/articles/424-testing-in-scrum-with-a-waterfall-interaction",
               :link => "http://scrumalliance.org/articles/424-testing-in-scrum-with-a-waterfall-interaction", # trims the link
               :title => "Testing in Scrum with a Waterfall Interaction",
+              :published => DateTime.parse("May 23, 2012 11:07:03 GMT"),
               :categories => [],
               :author => "",
               :description => "Sometimes, when testing user stories in Scrum, there's a final Waterfall  interaction to deal with. The scenario I present here is based on this  situation: a Scrum process with an interaction of sequential phases at  the end of the process to (re)test the whole developed functionality.  These sequential phases are mandatory for our organization, which  follows a Waterfall process for the releases of the product. So, for the  moment at least, we have to deal with this  and my experience is that  we aren't alone.",
@@ -196,6 +223,18 @@ describe FeedParser do
         end
       end
 
+      it "should set the published date if present" do
+        feed = FeedParser::Feed.new(feed_xml('smashingmagazine.atom.xml'))
+        item = feed.items.first
+        item.published.should == DateTime.parse("Jul 20, 2009 8:43:22 GMT")
+      end
+
+      it "should default the published date to the updated date if not present" do
+        feed = FeedParser::Feed.new(feed_xml('facebook.atom.xml'))
+        item = feed.items.first
+        item.published.should == DateTime.parse("Dec 30, 2011 17:00 GMT")
+      end
+
       {
         'gcal.atom.xml' => {
           :title => "dokaus.net",
@@ -213,6 +252,7 @@ describe FeedParser do
                 :guid => "urn:uuid:132266233552163",
                 :link => "http://developers.facebook.com/blog/post/614/",
                 :title => "Breaking Change: JavaScript SDK to oauth:true on December 13th",
+                :published => DateTime.parse("Dec 12, 2011 17:00 GMT"),
                 :categories=>[],
                 :author => "",
                 :description => "",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: feed_parser
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-29 00:00:00.000000000 Z
+date: 2013-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &2157337400 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2157337400
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &2157336900 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +37,15 @@ dependencies:
         version: '0.9'
   type: :development
   prerelease: false
-  version_requirements: *2157336900
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2157336400 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,7 +53,12 @@ dependencies:
         version: '2.10'
   type: :development
   prerelease: false
-  version_requirements: *2157336400
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '2.10'
 description: Rss and Atom feed parser with sanitizer support built on top of Nokogiri.
 email:
 - arttu.tervo@gmail.com
@@ -93,7 +108,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Rss and Atom feed parser