feedutils 0.1.0 → 0.2.0

data/Manifest.txt

@@ -7,3 +7,6 @@ lib/feedutils/builder/atom.rb
 lib/feedutils/builder/rss.rb
 lib/feedutils/utils.rb
 lib/feedutils/version.rb
+test/helper.rb
+test/test_atom.rb
+test/test_rss.rb

data/Rakefile

@@ -20,6 +20,8 @@ Hoe.spec 'feedutils' do
   self.extra_deps = [
     ['logutils', '>= 0.5']
   ]
+  
+  ###  todo: add fetcher dep for testing (e.g. development only)
 
   self.licenses = ['Public Domain']
 

data/lib/feedutils/builder/atom.rb

@@ -3,6 +3,8 @@ module FeedUtils
 
 class AtomFeedBuilder
 
+  include LogUtils::Logging
+
   def initialize( atom_feed )
     @feed = build_feed( atom_feed )
   end
@@ -39,19 +41,36 @@ class AtomFeedBuilder
     item.title     = atom_item.title.content
     item.url       = atom_item.link.href
 
+    logger.debug "  atom | item.title.content: >#{atom_item.title.content}< : #{atom_item.title.content.class.name}"
+    logger.debug "  atom | item.link.href: >#{atom_item.link.href}< : #{atom_item.link.href.class.name}"
+
+
     ## todo: check if updated or published present
     #    set 
-    item.updated    =  atom_item.updated.content.utc.strftime( "%Y-%m-%d %H:%M" )
-    item.published  =  item.updated  # fix: check if publshed set
+    item.updated    =  atom_item.updated.content  #  .utc.strftime( "%Y-%m-%d %H:%M" )
+
+
+    ## change time to utc if present? why? why not?
+
+    ### todo: use/try published first? why? why not?
+    logger.debug "  atom | item.updated  >#{atom_item.updated.content}< : #{atom_item.updated.content.class.name}"
+
+    # - todo/check: does it exist in atom format?
+    # item.published  =  item.updated  # fix: check if publshed set
 
     item.guid       =  atom_item.id.content
 
+    logger.debug "  atom | item.id.content: >#{atom_item.id.content}< : #{atom_item.id.content.class.name}"
 
     # todo: move logic to updater or something
     #  - not part of normalize
 
+
+    ## fix/todo:
+    #  also save/include full content in content
+
     if atom_item.summary
-      item.content = atom_item.summary.content
+      item.summary = atom_item.summary.content
     else
       if atom_item.content
         text  = atom_item.content.content.dup
@@ -59,21 +78,10 @@ class AtomFeedBuilder
         text = text.gsub( /<[^>]+>/, '' )
         text = text[ 0..400 ] # get first 400 chars
         ## todo: check for length if > 400 add ... at the end???
-        item.content = text
+        item.summary = text
       end
     end
 
-    puts "- #{atom_item.title.content}"
-    puts "  link >#{atom_item.link.href}<"
-    puts "  id (~guid) >#{atom_item.id.content}<"
-
-    ### todo: use/try published first? why? why not?
-    puts "  updated (~pubDate) >#{atom_item.updated.content}< >#{atom_item.updated.content.utc.strftime( "%Y-%m-%d %H:%M" )}< : #{atom_item.updated.content.class.name}"
-    puts
-
-    # puts "*** dump item:"
-    # pp item
-
     item
   end # method build_feed_item
 

data/lib/feedutils/builder/rss.rb

@@ -1,8 +1,13 @@
 
 module FeedUtils
 
+### todo/fix:
+#    rename to Rss20FeedBuilder?? or FeedBuilderRss20 ??
+
 class RssFeedBuilder
 
+  include LogUtils::Logging
+
   def initialize( rss_feed )
     @feed = build_feed( rss_feed )
   end
@@ -20,9 +25,30 @@ class RssFeedBuilder
   def build_feed( rss_feed )
     feed = Feed.new
     feed.object = rss_feed
-    feed.title  = rss_feed.channel.title
     feed.format = "rss #{rss_feed.rss_version}"
 
+    feed.title     = rss_feed.channel.title           # required
+    feed.url       = rss_feed.channel.link            # required
+    feed.summary   = rss_feed.channel.description     # required
+
+     # NOTE:
+     # All date-times in RSS conform
+     #   to the Date and Time Specification of RFC 822
+     #  e.g. Sun, 19 May 2012 15:21:36 GMT  or
+     #       Sat, 07 Sep 2013 00:00:01 GMT
+
+    feed.built     = rss_feed.channel.lastBuildDate   # optional
+    feed.published = rss_feed.channel.pubDate         # optional
+
+    logger.debug "  rss | channel.lastBuildDate: >#{rss_feed.channel.lastBuildDate}< : #{rss_feed.channel.lastBuildDate.class.name}"
+    logger.debug "  rss | channel.pubDate: >#{rss_feed.channel.pubDate}< : #{rss_feed.channel.pubDate.class.name}"
+
+
+    feed.generator = rss_feed.channel.generator       # optional
+
+    logger.debug "  rss | channel.generator: >#{rss_feed.channel.generator}< : #{rss_feed.channel.generator.class.name}"
+
+
     items = []
     rss_feed.items.each do |rss_item|
       items << build_feed_item( rss_item )
@@ -39,30 +65,52 @@ class RssFeedBuilder
 
     item.title     = rss_item.title
     item.url       = rss_item.link
-      
-    ## todo: check if updated or published present
-    #    set 
-    item.published = rss_item.pubDate.utc.strftime( "%Y-%m-%d %H:%M" )
-    item.updated   = item.published
-      
+
+## todo:
+##  check if feedburner:origLink present - if yes, use it for url/link
+##  example: use
+##  - <feedburner:origLink>http://www.rubyflow.com/items/9803-gotta-ruby-s-syntax</feedburner:origLink></item>
+##   instead of
+##  - <link>http://feedproxy.google.com/~r/Rubyflow/~3/Ym9Sltg_2_c/9803-gotta-ruby-s-syntax</link>
+
+
+    item.summary   = rss_item.description
+
+    logger.debug "  rss | item.title: >#{rss_item.title}< : #{rss_item.title.class.name}"
+    logger.debug "  rss | item.link: >#{rss_item.link}< : #{rss_item.link.class.name}"
+
+    # NOTE:
+    # All date-times in RSS conform
+    #   to the Date and Time Specification of RFC 822
+    #  e.g. Sun, 19 May 2012 15:21:36 GMT  or
+    #       Sat, 07 Sep 2013 00:00:01 GMT
+
+    item.published = rss_item.pubDate     # .utc.strftime( "%Y-%m-%d %H:%M" )
+
+    logger.debug "  rss | item.pubDate: >#{rss_item.pubDate}< : #{rss_item.pubDate.class.name}"
+ 
+    ## fix/todo: add
+    ## check for <content:encoded>
+    ##   full content  (example use e.g. in sitepoint/ruby/feed/)
     # content:  item.content_encoded,
   
     # if item.content_encoded.nil?
     # puts " using description for content"
-       
-    item.content  = rss_item.description
     # end
-        
+
+    ## fix/todo: check if rss_item.guid present? !!!!
     item.guid     = rss_item.guid.content
         
-    puts "- #{rss_item.title}"
-    puts "  link (#{rss_item.link})"
-    puts "  guid (#{rss_item.guid.content})"
-    puts "  pubDate >#{rss_item.pubDate}< >#{rss_item.pubDate.utc.strftime( "%Y-%m-%d %H:%M" )}< : #{rss_item.pubDate.class.name}"
-    puts
-
-    # puts "*** dump item:"
-    # pp item
+    logger.debug "  rss | item.guid.content: >#{rss_item.guid.content}< : #{rss_item.guid.content.class.name}"
+
+    ### todo: add support or authors (incl. dc:creator)
+    ##  <dc:creator>Dhaivat Pandya</dc:creator>
+
+    #  todo: categories
+    # <category><![CDATA[Gems]]></category>
+    # <category><![CDATA[Ruby]]></category>
+    # <category><![CDATA[Ruby on Rails]]></category>
+
 
     item
   end # method build_feed_item_from_rss

data/lib/feedutils/utils.rb

@@ -7,10 +7,67 @@ module FeedUtils
 
     attr_accessor :format   # e.g. atom|rss 2.0|etc.
     attr_accessor :title
-    attr_accessor :title_type  # e.g. text|html  (optional) -use - why?? why not??
+    attr_accessor :title_type  # e.g. text|html|html-escaped  (optional) -use - why?? why not??
+    attr_accessor :url
 
     attr_accessor :items
 
+    attr_accessor :summary        # e.g. description (rss)
+    attr_accessor :summary_type   # e.g. text|html|html-escaped
+    attr_accessor :title2         # e.g. subtitle (atom)
+    attr_accessor :title2_type    # e.g. text|html|html-escaped
+
+    attr_accessor :published
+    attr_accessor :updated
+    attr_accessor :built
+    
+    attr_accessor :generator
+
+
+    def title2?
+      @title2.nil? == false
+    end
+
+    def summary?
+      @summary.nil? == false
+    end
+
+    def built?
+      @built.nil? == false
+    end
+    
+    def updated?
+      @updated.nil? == false
+    end
+    
+    def published?
+      @published.nil? == false
+    end
+
+
+    def summary
+      # no summary? try/return title2
+      if summary?
+        @summary
+      else
+        @title2
+      end
+    end
+
+    def published
+      # no published date? try/return updated or built
+      if published?
+        @published
+      elsif updated?
+        @updated
+      else
+        @built
+      end
+    end
+
+    ## fix:
+    #  add pretty printer/inspect (exclude object)
+
   end  # class Feed
 
 
@@ -18,24 +75,65 @@ module FeedUtils
     attr_accessor :object   # orginal object (e.g RSS item or ATOM entry etc.)
 
     attr_accessor :title
-    attr_accessor :title_type    # optional for now (text|html) - not yet set
+    attr_accessor :title_type    # optional for now (text|html|html-escaped) - not yet set
     attr_accessor :url      # todo: rename to link (use alias) ??
     attr_accessor :content
-    attr_accessor :content_type  # optional for now (text|html) - not yet set
+    attr_accessor :content_type  # optional for now (text|html|html-escaped|binary-base64) - not yet set
+    attr_accessor :summary
+    attr_accessor :summary_type  # optional for now (text|html|html-escaped) - not yet set
 
 ## todo: add summary (alias description)  ???
 ## todo: add author/authors
 ## todo: add category/categories
 
-    attr_accessor :updated
     attr_accessor :published
+    attr_accessor :updated
 
     attr_accessor :guid     # todo: rename to id (use alias) ??
+
+
+    def summary?
+      @summary.nil? == false
+    end
+
+    def content?
+      @content.nil? == false
+    end
+    
+    def published?
+      @published.nil? == false
+    end
+
+    def updated?
+      @updated.nil? == false
+    end
+    
+    def content
+      # no content? try/return summary
+      if content?
+        @content
+      else
+        @summary
+      end
+    end
+
+    def published
+      # no published date? try/return updated
+      if published?
+        @published
+      else
+        @updated
+      end
+    end
+
+
   end  # class Item
 
 
   class Parser
 
+    include LogUtils::Logging
+
     ### Note: lets keep/use same API as RSS::Parser for now
     def initialize( xml )
       @xml = xml
@@ -49,7 +147,7 @@ module FeedUtils
       puts "Parsing feed..."
       feed_wild = parser.parse  # not yet normalized
 
-      puts "  feed.class=#{feed_wild.class.name}"
+      logger.debug "  feed.class=#{feed_wild.class.name}"
 
       if feed_wild.is_a?( RSS::Atom::Feed )
         feed = AtomFeedBuilder.build( feed_wild )
@@ -60,7 +158,12 @@ module FeedUtils
       puts "== #{feed.format} / #{feed.title} =="
       feed # return new (normalized) feed
     end
- 
+    
+    ### convenience class/factory method
+    def self.parse( xml, opts={} )
+      self.new( xml ).parse
+    end
+
   end  # class Parser
 
 

data/lib/feedutils/version.rb

@@ -1,4 +1,4 @@
 
 module FeedUtils
-   VERSION = '0.1.0'
+   VERSION = '0.2.0'
 end

data/test/helper.rb

@@ -0,0 +1,25 @@
+## $:.unshift(File.dirname(__FILE__))
+
+
+## minitest setup
+
+# require 'minitest/unit'
+require 'minitest/autorun'
+
+# include MiniTest::Unit  # lets us use TestCase instead of MiniTest::Unit::TestCase
+
+require 'logutils'
+require 'fetcher'
+
+## our own code
+
+require 'feedutils'
+
+
+LogUtils::Logger.root.level = :debug
+
+def parse_feed( feed_url )
+  xml = Fetcher.read( feed_url )
+
+  FeedUtils::Parser.parse( xml )
+end

data/test/test_atom.rb

@@ -0,0 +1,16 @@
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_rss.rb
+#  or better
+#     rake test
+
+require 'helper'
+
+class TestAtom < MiniTest::Unit::TestCase
+
+  def test_rubyonrails
+    feed = parse_feed( 'http://weblog.rubyonrails.org/feed/atom.xml' )
+    assert( feed.format == 'atom' )
+  end
+
+end

data/test/test_rss.rb

@@ -0,0 +1,23 @@
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_rss.rb
+#  or better
+#     rake test
+
+require 'helper'
+
+class TestRss < MiniTest::Unit::TestCase
+
+
+  def test_rubyflow
+    feed = parse_feed( 'http://feeds.feedburner.com/Rubyflow?format=xml' )
+    assert( feed.format == 'rss 2.0' )
+  end
+
+  def test_sitepointruby
+    feed = parse_feed( 'http://www.sitepoint.com/ruby/feed/' )
+    assert( feed.format == 'rss 2.0' )
+  end
+
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: feedutils
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-19 00:00:00.000000000 Z
+date: 2013-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logutils
-  requirement: &73814860 !ruby/object:Gem::Requirement
+  requirement: &68865360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0.5'
   type: :runtime
   prerelease: false
-  version_requirements: *73814860
+  version_requirements: *68865360
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &73813400 !ruby/object:Gem::Requirement
+  requirement: &68864940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: '3.10'
   type: :development
   prerelease: false
-  version_requirements: *73813400
+  version_requirements: *68864940
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &73812180 !ruby/object:Gem::Requirement
+  requirement: &68864560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +43,7 @@ dependencies:
         version: '3.3'
   type: :development
   prerelease: false
-  version_requirements: *73812180
+  version_requirements: *68864560
 description: feedutils - web feed parser and normalizer (RSS 2.0, Atom, etc.)
 email: webslideshow@googlegroups.com
 executables: []
@@ -60,6 +60,10 @@ files:
 - lib/feedutils/builder/rss.rb
 - lib/feedutils/utils.rb
 - lib/feedutils/version.rb
+- test/helper.rb
+- test/test_atom.rb
+- test/test_rss.rb
+- .gemtest
 homepage: https://github.com/rubylibs/feedutils
 licenses:
 - Public Domain
@@ -87,4 +91,6 @@ rubygems_version: 1.8.17
 signing_key: 
 specification_version: 3
 summary: feedutils - web feed parser and normalizer (RSS 2.0, Atom, etc.)
-test_files: []
+test_files:
+- test/test_atom.rb
+- test/test_rss.rb