feedupdater 0.2.0 → 0.2.1

data/CHANGELOG

@@ -1,3 +1,7 @@
+== FeedUpdater 0.2.1
+ * no longer attempts to update feeds that have been updated very recently
+ * fixed logging levels
+ * added on_begin block
 == FeedUpdater 0.2.0
  * multithreaded
  * unchanged feeds will no longer call on_update

data/bin/feed_updater

@@ -1,4 +1,4 @@
-#!/usr/local/bin/ruby
+#!/usr/bin/env ruby
 
 #--
 # Copyright (c) 2005 Robert Aman

data/example/custom_updater.rb

@@ -1,12 +1,22 @@
 class CustomUpdater < FeedTools::FeedUpdater
+  on_begin do
+#     self.feed_href_list = [
+#       "http://www.gigaom.com/feed/rss2/",
+#       "http://feeds.feedburner.com/ManeuverMarketingCommunique",
+#       "http://www.afp.com/english/rss/stories.xml",
+#       "http://www.railheaddesign.com/rss/railhead.xml",
+#       "http://www.nateanddi.com/rssfeed.xml"
+#     ]
+  end
+  
   on_update do |feed, seconds|
-    logger.info("Loaded '#{feed.href}'.")
-    logger.info("=> Updated (#{feed.title}) in #{seconds} seconds.")
+    self.logger.info("Loaded '#{feed.href}'.")
+    self.logger.info("=> Updated (#{feed.title}) in #{seconds} seconds.")
   end
   
   on_error do |href, error|
-    logger.info("Error updating '#{href}':")
-    logger.info(error)
+    self.logger.info("Error updating '#{href}':")
+    self.logger.info(error)
   end
 
   on_complete do |updated_feed_hrefs|

data/lib/feed_updater.rb

@@ -64,7 +64,7 @@ class FeedUpdaterLogger < Logger
   alias_method :old_log, :log
   def log(level, message)
     if defined?(@prefix) && @prefix != nil
-      self.old_log(level, self.prefix + message)
+      self.old_log(level, "#{self.prefix}#{message}")
     else
       self.old_log(level, message)
     end
@@ -94,6 +94,14 @@ end
 module FeedTools
   # A simple daemon for scheduled updating of feeds.
   class FeedUpdater
+    # Declares an on_begin event.  The given block will be called before the
+    # update sequence runs to allow for any setup required.  The block is
+    # not passed any arguments.
+    def self.on_begin(&block)
+      raise "No block supplied for on_begin." if block.nil?
+      @@on_begin = block
+    end
+
     # Declares an on_update event.  The given block will be called after
     # every feed update.  The block is passed the feed object that was loaded
     # and the time it took in seconds to successfully load it.
@@ -325,18 +333,25 @@ module FeedTools
           
           # The main feed update loop.
           loop do
-            result = Benchmark.measure do
-              self.update_feeds()
-            end
-            self.logger.info(
-              "#{@feed_href_list.size} feed(s) updated " +
-              "in #{result.real.round} seconds.")
-            if !defined?(@feed_href_list_override) ||
-                !@feed_href_list_override
-              @feed_href_list = nil
+            result = nil
+            sleepy_time = 1.hour
+            begin
+              result = Benchmark.measure do
+                self.update_feeds()
+              end
+              self.logger.info(
+                "#{@feed_href_list.size} feed(s) updated " +
+                "in #{result.real.round} seconds.")
+              sleepy_time = 1.hour - result.real.round
+            rescue Exception => error
+              self.logger.error("Feed update sequence errored out.")
+              self.logger.error(error.class.name + ": " + error.message)
+              self.logger.error("\n" + error.backtrace.join("\n").to_s)
             end
+
+            @feed_href_list = nil
+            @feed_href_list_override = false
             ObjectSpace.garbage_collect()
-            sleepy_time = 1.hour - result.real.round
             if sleepy_time > 0
               self.logger.info(
                 "Sleeping for #{(sleepy_time / 60.0).round} minutes...")
@@ -419,18 +434,39 @@ module FeedTools
     
     # Updates all of the feeds.
     def update_feeds()
+      self.logger.level = 0
       self.logger.prefix = "FeedUpdater".ljust(20)
       ObjectSpace.garbage_collect()
+      if defined?(@@on_begin) && @@on_begin != nil
+        self.logger.info("Running custom startup event...")
+        self.cloaker(&(@@on_begin)).bind(self).call()
+      end
       if defined?(@feed_href_list_override) && @feed_href_list_override
+        self.logger.info("Using custom feed list...")
         self.feed_href_list()
       else
         self.logger.info("Loading default feed list...")
-        @feed_href_list =
-          (FeedTools.feed_cache.find(:all).collect do |cache_object|
-            cache_object.href
-          end)
+        begin
+          expire_time = (Time.now - 1.hour).utc
+          expire_time_string = sprintf('%04d-%02d-%02d %02d:%02d:%02d',
+            expire_time.year, expire_time.month, expire_time.day,
+            expire_time.hour, expire_time.min, expire_time.sec)
+          @feed_href_list =
+            FeedTools.feed_cache.connection.execute(
+              "SELECT href FROM cached_feeds WHERE " +
+              "last_retrieved < '#{expire_time_string}'").to_a.flatten
+        rescue Exception
+          self.logger.warn("Default feed list failed, using fallback.")
+          @feed_href_list =
+            FeedTools.feed_cache.find(:all).collect do |feed|
+              feed.href
+            end
+          self.logger.warn(
+            "Fallback succeeded. Custom feed list override recommended.")
+        end
       end
       self.logger.info("Updating #{@feed_href_list.size} feed(s)...")
+      self.logger.level = self.updater_options[:log_level]
       ObjectSpace.garbage_collect()
       
       threads = []
@@ -487,52 +523,56 @@ module FeedTools
           href_list = thread_slices[Thread.current.thread_id]
 
           for i in 0...href_list.size
-            href = nil
             progress = nil
             mutex.synchronize do
-              href = href_list[i]
+              Thread.current.href = href_list[i]
               Thread.current.progress =
-                (href_list.index(href).to_f / href_list.size.to_f) * 100
+                (href_list.index(Thread.current.href).to_f /
+                  href_list.size.to_f) * 100
               progress = sprintf("%.2f", Thread.current.progress)
             end
             begin
               begin
-                feed = nil
+                Thread.current.feed = nil
                 feed_load_benchmark = Benchmark.measure do
-                  feed = FeedTools::Feed.open(href)
+                  Thread.current.feed =
+                    FeedTools::Feed.open(Thread.current.href)
                 end
                 Thread.pass
-                if feed.live?
+                if Thread.current.feed.live?
                   unless @@on_update.nil?
                     mutex.synchronize do
                       progress = sprintf("%.2f", Thread.current.progress)
-                      self.logger.prefix = ("Thread #{Thread.current.thread_id} " +
-                        "(#{progress}%)"
-                        ).ljust(20)
+                      self.logger.prefix = 
+                        ("Thread #{Thread.current.thread_id} (#{progress}%)"
+                          ).ljust(20)
                       self.cloaker(&(@@on_update)).bind(self).call(
-                        feed, feed_load_benchmark.real)
+                        Thread.current.feed, feed_load_benchmark.real)
                     end
                   end
                 else
                   mutex.synchronize do
                     progress = sprintf("%.2f", Thread.current.progress)
-                    self.logger.prefix = ("Thread #{Thread.current.thread_id} " +
-                      "(#{progress}%)"
-                      ).ljust(20)
+                    self.logger.prefix = 
+                      ("Thread #{Thread.current.thread_id} (#{progress}%)"
+                        ).ljust(20)
                     self.logger.info(
-                      "'#{href}' unchanged, skipping.")
+                      "'#{Thread.current.href}' unchanged " +
+                      "or unavailable, skipping.")
                   end
                 end
               rescue Exception => error
                 mutex.synchronize do
                   progress = sprintf("%.2f", Thread.current.progress)
-                  self.logger.prefix = ("Thread #{Thread.current.thread_id} " +
-                    "(#{progress}%)"
-                    ).ljust(20)
+                  self.logger.prefix = 
+                    ("Thread #{Thread.current.thread_id} (#{progress}%)"
+                      ).ljust(20)
                   if @@on_error != nil
-                    self.cloaker(&(@@on_error)).bind(self).call(href, error)
+                    self.cloaker(&(@@on_error)).bind(self).call(
+                      Thread.current.href, error)
                   else
-                    self.logger.error("Error updating '#{href}':")
+                    self.logger.error(
+                      "Error updating '#{Thread.current.href}':")
                     self.logger.error(error.class.name + ": " + error.message)
                     self.logger.error(error.class.backtrace)
                   end
@@ -545,7 +585,7 @@ module FeedTools
                   "(#{progress}%)"
                   ).ljust(20)
                 self.logger.fatal("Critical unhandled error.")
-                self.logger.fatal("Error updating '#{href}':")
+                self.logger.fatal("Error updating '#{Thread.current.href}':")
                 self.logger.fatal(error.class.name + ": " + error.message)
                 self.logger.fatal(error.class.backtrace)
               end
@@ -558,6 +598,8 @@ module FeedTools
         class <<updater_thread
           attr_accessor :thread_id
           attr_accessor :progress
+          attr_accessor :href
+          attr_accessor :feed
         end
         updater_thread.thread_id = i
       end
@@ -586,7 +628,9 @@ module FeedTools
       unless @@on_complete.nil?
         self.cloaker(&(@@on_complete)).bind(self).call(@feed_href_list)
       end
+      self.logger.level = 0
       self.logger.info("Finished updating.")
+      self.logger.level = self.updater_options[:log_level]
     end
   end
 end

data/lib/feed_updater/version.rb

@@ -2,7 +2,7 @@ module FeedTools
   module FEED_UPDATER_VERSION #:nodoc:
     MAJOR = 0
     MINOR = 2
-    TINY  = 0
+    TINY  = 1
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.8.11
 specification_version: 1
 name: feedupdater
 version: !ruby/object:Gem::Version 
-  version: 0.2.0
-date: 2006-04-17 00:00:00 -07:00
+  version: 0.2.1
+date: 2006-04-20 00:00:00 -07:00
 summary: Automatic feed updater daemon.
 require_paths: 
   - lib
@@ -32,6 +32,7 @@ files:
   - bin
   - CHANGELOG
   - config
+  - doc
   - example
   - lib
   - log