feedtools 0.2.11 → 0.2.12

data/CHANGELOG

@@ -1,3 +1,12 @@
+== FeedTools 0.2.12
+ * schema change!
+ * fixed missing cache connection breaking redirects
+ * fixed redirects creating duplicate cache entry
+ * exception now raised if cache_only option is set but the cache is disabled
+ * additional unit tests to verify that the cache works with redirects
+ * fixed feed expiration bugs with expire!
+ * xml_data renamed to feed_data
+ * feed_data_type method added
 == FeedTools 0.2.11
  * ruby -w shouldn't produce nearly as many warnings for feed_tools.rb anymore
  * you can now force the open method to only pull from the cache

data/db/schema.mysql.sql

@@ -4,7 +4,8 @@
     `url`             varchar(255) default NULL,
     `title`           varchar(255) default NULL,
     `link`            varchar(255) default NULL,
-    `xml_data`        longtext default NULL,
+    `feed_data`       longtext default NULL,
+    `feed_data_type`  varchar(20) default NULL,
     `http_headers`    text default NULL,
     `last_retrieved`  datetime default NULL,
     PRIMARY KEY  (`id`)

data/db/schema.postgresql.sql

@@ -4,7 +4,8 @@
     url               varchar(255) default NULL,
     title             varchar(255) default NULL,
     link              varchar(255) default NULL,
-    xml_data          text default NULL,
+    feed_data         text default NULL,
+    feed_data_type    varchar(20) default NULL,
     http_headers      text default NULL,
     last_retrieved    timestamp default NULL
   );

data/db/schema.sqlite.sql

@@ -4,7 +4,8 @@
     url               VARCHAR(255) DEFAULT NULL,
     title             VARCHAR(255) DEFAULT NULL,
     link              VARCHAR(255) DEFAULT NULL,
-    xml_data          TEXT DEFAULT NULL,
+    feed_data         TEXT DEFAULT NULL,
+    feed_data_type    VARCHAR(20) DEFAULT NULL,
     http_headers      TEXT DEFAULT NULL,
     last_retrieved    DATETIME DEFAULT NULL
   );

data/lib/feed_tools.rb

@@ -32,7 +32,7 @@ FEED_TOOLS_ENV = ENV['FEED_TOOLS_ENV'] ||
                  ENV['RAILS_ENV'] ||
                  'production' # :nodoc:
 
-FEED_TOOLS_VERSION = "0.2.11"
+FEED_TOOLS_VERSION = "0.2.12"
 
 $:.unshift(File.dirname(__FILE__))
 $:.unshift(File.dirname(__FILE__) + "/feed_tools/vendor")

data/lib/feed_tools/database_feed_cache.rb

@@ -71,7 +71,7 @@ module FeedTools
     def DatabaseFeedCache.table_exists?
       begin
         ActiveRecord::Base.connection.execute "select id, url, title, " +
-          "link, xml_data, http_headers, last_retrieved " +
+          "link, feed_data, feed_data_type, http_headers, last_retrieved " +
           "from #{self.table_name()} limit 1"
       rescue ActiveRecord::StatementInvalid
         return false

data/lib/feed_tools/feed.rb

@@ -101,7 +101,8 @@ module FeedTools
       @cache_object = nil
       @http_headers = nil
       @xml_doc = nil
-      @xml_data = nil
+      @feed_data = nil
+      @feed_data_type = nil
       @root_node = nil
       @channel_node = nil
       @url = nil
@@ -134,6 +135,11 @@ module FeedTools
                        options.keys)
       options = { :cache_only => false }.merge(options)
           
+      if options[:cache_only] && FeedTools.feed_cache.nil?
+        raise(ArgumentError, "There is currently no caching mechanism set. " +
+          "Cannot retrieve cached feeds.")
+      end
+      
       # clean up the url
       url = FeedTools.normalize_url(url)
 
@@ -269,15 +275,28 @@ module FeedTools
               
                 # Find out if we've already seen the url we've been
                 # redirected to.
-                cached_feed = FeedTools::Feed.open(new_location,
-                  :cache_only => true)
                 found_redirect = false
-                if cached_feed.cache_object.new_record? != true
-                  unless cached_feed.expired?
-                    self.url = cached_feed.url
-                    @live = false
-                    found_redirect = true
+                begin
+                  cached_feed = FeedTools::Feed.open(new_location,
+                    :cache_only => true)
+                  if cached_feed.cache_object != nil &&
+                      cached_feed.cache_object.new_record? != true
+                    unless cached_feed.expired?
+                      # Copy the cached state, starting with the url
+                      self.url = cached_feed.url
+                      self.title = cached_feed.title
+                      self.link = cached_feed.link
+                      self.feed_data = cached_feed.feed_data
+                      self.feed_data_type = cached_feed.feed_data_type
+                      self.last_retrieved = cached_feed.last_retrieved
+                      self.http_headers = cached_feed.http_headers
+                      self.cache_object = cached_feed.cache_object
+                      @live = false
+                      found_redirect = true
+                    end
                   end
+                rescue
+                  # If anything goes wrong, ignore it.
                 end
                 unless found_redirect
                   # TODO: deal with stupid people using relative urls
@@ -337,29 +356,29 @@ module FeedTools
               self.http_headers[header.first.downcase] = header.last
             end
             self.last_retrieved = Time.now
-            self.xml_data = self.http_response.body
+            self.feed_data = self.http_response.body
           end
         rescue FeedAccessError
           @live = false
-          if self.xml_data.nil?
+          if self.feed_data.nil?
             raise
           end
         rescue Timeout::Error
-          # if we time out, do nothing, it should fall back to the xml_data
+          # if we time out, do nothing, it should fall back to the feed_data
           # stored in the cache.
           @live = false
-          if self.xml_data.nil?
+          if self.feed_data.nil?
             raise
           end
         rescue Errno::ECONNRESET
           # if the connection gets reset by peer, oh well, fall back to the
-          # xml_data stored in the cache
+          # feed_data stored in the cache
           @live = false
-          if self.xml_data.nil?
+          if self.feed_data.nil?
             raise
           end
         rescue => error
-          # heck, if anything at all bad happens, fall back to the xml_data
+          # heck, if anything at all bad happens, fall back to the feed_data
           # stored in the cache.
         
           # If we can, get the HTTPResponse...
@@ -381,7 +400,7 @@ module FeedTools
             end
           end
           @live = false
-          if self.xml_data.nil?
+          if self.feed_data.nil?
             if error.respond_to?(:response) &&
                 error.response.respond_to?(:response_chain)
               redirects = error.response.response_chain.map do |pair|
@@ -409,7 +428,8 @@ module FeedTools
             @http_response = nil
             @http_headers = {}
             self.last_retrieved = Time.now
-            self.xml_data = file.read
+            self.feed_data = file.read
+            self.feed_data_type = :xml
           end
         rescue
           @live = false
@@ -437,35 +457,60 @@ module FeedTools
       return @http_headers
     end
   
-    # Returns the feed's raw xml data.
-    def xml_data
-      if @xml_data.nil?
+    # Returns the feed's raw data.
+    def feed_data
+      if @feed_data.nil?
         unless self.cache_object.nil?
-          @xml_data = self.cache_object.xml_data
+          @feed_data = self.cache_object.feed_data
         end
       end
-      return @xml_data
+      return @feed_data
     end
   
-    # Sets the feed's xml data.
-    def xml_data=(new_xml_data)
-      @xml_data = new_xml_data
+    # Sets the feed's data.
+    def feed_data=(new_feed_data)
+      @feed_data = new_feed_data
+      unless self.cache_object.nil?
+        self.cache_object.feed_data = new_feed_data
+      end
+    end
+    
+    # Returns the data type of the feed
+    # Possible values:
+    # * :xml
+    # * :yaml
+    # * :text
+    def feed_data_type
+      if @feed_data_type.nil?
+        # Right now, nothing else is supported
+        @feed_data_type = :xml
+      end
+      return @feed_data_type
+    end
+
+    # Sets the feed's data type.
+    def feed_data_type=(new_feed_data_type)
+      @feed_data_type = new_feed_data_type
       unless self.cache_object.nil?
-        self.cache_object.xml_data = new_xml_data
+        self.cache_object.feed_data_type = new_feed_data_type
       end
     end
   
-    # Returns a REXML Document of the xml_data
+    # Returns a REXML Document of the feed_data
     def xml
-      if @xml_doc.nil?
-        begin
-          # TODO: :ignore_whitespace_nodes => :all
-          # Add that?
-          # ======================================
-          @xml_doc = Document.new(xml_data)
-        rescue
-          # Something failed, attempt to repair the xml with htree.
-          @xml_doc = HTree.parse(xml_data).to_rexml
+      if self.feed_data_type != :xml
+        @xml_doc = nil
+      else
+        if @xml_doc.nil?
+          begin
+            # TODO: :ignore_whitespace_nodes => :all
+            # Add that?
+            # ======================================
+            @xml_doc = Document.new(feed_data)
+          rescue
+            # Something failed, attempt to repair the xml with htree.
+            @xml_doc = HTree.parse(feed_data).to_rexml
+          end
         end
       end
       return @xml_doc
@@ -539,8 +584,10 @@ module FeedTools
     # title=
     # link
     # link=
-    # xml_data
-    # xml_data=
+    # feed_data
+    # feed_data=
+    # feed_data_type
+    # feed_data_type=
     # etag
     # etag=
     # last_modified
@@ -657,7 +704,7 @@ module FeedTools
   
     # Returns the feed url.
     def url
-      if @url.nil? && self.xml_data != nil
+      if @url.nil? && self.feed_data != nil
         @url = XPath.first(channel_node, "link[@rel='self']/@href").to_s
         @url = nil if @url == ""
       end
@@ -1675,7 +1722,8 @@ module FeedTools
         if raw_items != nil
           for item_node in raw_items
             new_item = FeedItem.new
-            new_item.xml_data = item_node.to_s
+            new_item.feed_data = item_node.to_s
+            new_item.feed_data_type = self.feed_data_type
             new_item.feed = self
             @items << new_item
           end
@@ -1748,7 +1796,7 @@ module FeedTools
   
     # Forces this feed to expire.
     def expire!
-      self.last_retrieved = Time.mktime(1970)
+      self.last_retrieved = Time.mktime(1970).gmtime
       self.save
     end
 
@@ -2014,15 +2062,16 @@ module FeedTools
         raise "Caching is currently disabled.  Cannot save to cache."
       elsif self.url.nil?
         raise "The url field must be set to save to the cache."
-      elsif self.xml_data.nil?
-        raise "The xml_data field must be set to save to the cache."
       elsif self.cache_object.nil?
         raise "The cache_object is currently nil.  Cannot save to cache."
       else
         self.cache_object.url = self.url
-        self.cache_object.title = self.title
-        self.cache_object.link = self.link
-        self.cache_object.xml_data = self.xml_data
+        unless self.feed_data.nil?
+          self.cache_object.title = self.title
+          self.cache_object.link = self.link
+          self.cache_object.feed_data = self.feed_data
+          self.cache_object.feed_data_type = self.feed_data_type.to_s
+        end
         unless self.http_response.nil?
           self.cache_object.http_headers = self.http_headers.to_yaml
         end

data/lib/feed_tools/feed_item.rb

@@ -126,6 +126,8 @@ module FeedTools
     def initialize
       super
       @feed = nil
+      @feed_data = nil
+      @feed_data_type = nil
       @xml_doc = nil
       @root_node = nil
       @title = nil
@@ -143,23 +145,37 @@ module FeedTools
       @feed = new_feed
     end
 
-    # Returns the feed item's raw xml data.
-    def xml_data
-      return @xml_data
+    # Returns the feed item's raw data.
+    def feed_data
+      return @feed_data
     end
 
-    # Sets the feed item's xml data.
-    def xml_data=(new_xml_data)
-      @xml_data = new_xml_data
+    # Sets the feed item's data.
+    def feed_data=(new_feed_data)
+      @feed_data = new_feed_data
     end
 
-    # Returns a REXML Document of the xml_data
+    # Returns the feed item's data type.
+    def feed_data_type
+      return @feed_data_type
+    end
+
+    # Sets the feed item's data type.
+    def feed_data_type=(new_feed_data_type)
+      @feed_data_type = new_feed_data_type
+    end
+
+    # Returns a REXML Document of the feed_data
     def xml
-      if @xml_doc.nil?
-        # TODO: :ignore_whitespace_nodes => :all
-        # Add that?
-        # ======================================
-        @xml_doc = Document.new(xml_data)
+      if self.feed_data_type != :xml
+        @xml_doc = nil
+      else
+        if @xml_doc.nil?
+          # TODO: :ignore_whitespace_nodes => :all
+          # Add that?
+          # ======================================
+          @xml_doc = Document.new(self.feed_data)
+        end
       end
       return @xml_doc
     end

data/rakefile

@@ -7,7 +7,7 @@ require 'rake/gempackagetask'
 require 'rake/contrib/rubyforgepublisher'
 
 PKG_NAME      = 'feedtools'
-PKG_VERSION   = '0.2.11'
+PKG_VERSION   = '0.2.12'
 PKG_FILE_NAME = "#{PKG_NAME}-#{PKG_VERSION}"
 
 RELEASE_NAME  = "REL #{PKG_VERSION}"

data/test/amp_test.rb

@@ -456,7 +456,7 @@ class AmpTest < Test::Unit::TestCase
   
   def test_amp_65
     feed = FeedTools::Feed.new
-    feed.xml_data = <<-FEED
+    feed.feed_data = <<-FEED
       <feed version="0.3">
         <title>&lt;strong>1 &amp;amp; 2 &amp; 3&lt;/strong></title>
         <tagline>&lt;strong>1 &amp;amp; 2 &amp; 3&lt;/strong></tagline>

data/test/cache_test.rb

@@ -7,7 +7,7 @@ class CacheTest < Test::Unit::TestCase
   end
 
   def test_database_connection
-    # turn the cache on for this test
+    # Ensure the cache is on for this test
     FeedTools.feed_cache = FeedTools::DatabaseFeedCache
 
     unless FeedTools.feed_cache.nil?
@@ -16,8 +16,41 @@ class CacheTest < Test::Unit::TestCase
     else
       puts "\nSkipping cache test since the cache is still disabled.\n"
     end
-
-    # turn the cache back off
+  end
+  
+  def test_redirects_when_cache_disabled
+    # Turn the cache off for this test
     FeedTools.feed_cache = nil
+
+    # We just want to make sure there's no exception
+    slashdot_feed = FeedTools::Feed.open('http://www.slashdot.org/index.rss')
+    
+    # Turn the cache back on
+    FeedTools.feed_cache = FeedTools::DatabaseFeedCache
+  end
+
+  def test_redirects_when_cache_enabled
+    # Ensure the cache is on for this test
+    FeedTools.feed_cache = FeedTools::DatabaseFeedCache
+    
+    slashdot_feed = FeedTools::Feed.open('http://www.slashdot.org/index.rss')
+    assert(slashdot_feed.feed_data != nil, "No content retrieved.")
+    slashdot_feed.expire!
+    assert_equal(true, slashdot_feed.expired?)
+    slashdot_feed = FeedTools::Feed.open('http://www.slashdot.org/index.rss')
+    assert(slashdot_feed.feed_data != nil, "No content retrieved.")
+    assert_equal(true, slashdot_feed.live?)
+    slashdot_feed = FeedTools::Feed.open('http://www.slashdot.org/index.rss')
+    assert(slashdot_feed.feed_data != nil, "No content retrieved.")
+    assert_equal(false, slashdot_feed.live?)    
+    slashdot_feed.expire!
+    slashdot_feed.expire!
+    slashdot_feed = FeedTools::Feed.open('http://www.slashdot.org/index.rss')
+    assert(slashdot_feed.feed_data != nil, "No content retrieved.")
+    assert_equal(true, slashdot_feed.live?)
+    FeedTools::Feed.open(slashdot_feed.url)
+    
+    entries = FeedTools::DatabaseFeedCache.find_all_by_url(slashdot_feed.url)
+    assert_equal(1, entries.size)
   end
 end

data/test/cdf_test.rb

@@ -8,7 +8,7 @@ class CdfTest < Test::Unit::TestCase
 
   def test_feed_title
     feed = FeedTools::Feed.new
-    feed.xml_data = <<-FEED
+    feed.feed_data = <<-FEED
       <CHANNEL>
         <TITLE>Example Title</TITLE>
       </CHANNEL>
@@ -36,7 +36,7 @@ class CdfTest < Test::Unit::TestCase
   
   def test_feed_href
     feed = FeedTools::Feed.new
-    feed.xml_data = <<-FEED
+    feed.feed_data = <<-FEED
       <CHANNEL HREF="http://www.example.com/">
       </CHANNEL>
     FEED
@@ -54,7 +54,7 @@ class CdfTest < Test::Unit::TestCase
 
   def test_feed_images
     feed = FeedTools::Feed.new
-    feed.xml_data = <<-FEED
+    feed.feed_data = <<-FEED
       <CHANNEL>
         <LOGO HREF="http://www.example.com/exampleicon.gif" STYLE="ICON" />
         <LOGO HREF="http://www.example.com/exampleimage.gif" STYLE="IMAGE" />
@@ -100,7 +100,7 @@ class CdfTest < Test::Unit::TestCase
 
   def test_feed_item_images
     feed = FeedTools::Feed.new
-    feed.xml_data = <<-FEED
+    feed.feed_data = <<-FEED
       <CHANNEL>
         <ITEM HREF="http://www.example.com/item">
           <LOGO HREF="http://www.example.com/exampleicon.gif" STYLE="ICON" />

data/test/nonstandard_test.rb

@@ -8,7 +8,7 @@ class NonStandardTest < Test::Unit::TestCase
   
   def test_xss_strict
     feed = FeedTools::Feed.new
-    feed.xml_data = <<-FEED
+    feed.feed_data = <<-FEED
       <?xml version="1.0" encoding="iso-8859-1"?>
       <rss version="2.0/XSS-strict">
         <channel>
@@ -66,7 +66,7 @@ class NonStandardTest < Test::Unit::TestCase
   def test_rss_30_lite
     # Delusions of grandeur...
     feed = FeedTools::Feed.new
-    feed.xml_data = <<-FEED
+    feed.feed_data = <<-FEED
       <?xml version="1.0" encoding="UTF-8"?>
       <rss version="3.0" type="lite"
            source="http://www.rss3.org/files/liteSample.rss">

metadata

@@ -3,7 +3,7 @@ rubygems_version: 0.8.11
 specification_version: 1
 name: feedtools
 version: !ruby/object:Gem::Version 
-  version: 0.2.11
+  version: 0.2.12
 date: 2005-09-27 00:00:00 -04:00
 summary: "Parsing, generation, and caching system for xml news feeds."
 require_paths: