feedupdater 0.1.0 → 0.2.0

data/CHANGELOG

@@ -1,2 +1,5 @@
+== FeedUpdater 0.2.0
+ * multithreaded
+ * unchanged feeds will no longer call on_update
 == FeedUpdater 0.1.0
  * initial version

data/config/feed_updater.yml

@@ -5,8 +5,15 @@ start_delay: false
 # See the documentation for details.
 load_script: example/custom_updater.rb
 
+# Keep this set to 1 unless you're certain you need to set it higher.
+# Never set it higher than 1 if you're on a shared server.
+threads: 5
+
 # This is the path to store the pid files.
 pid_file_path: config
 
 # This is the path to store the log files.
 log_file_path: log
+
+# Log level
+log_level: 0

data/lib/feed_updater/version.rb

@@ -1,7 +1,7 @@
 module FeedTools
   module FEED_UPDATER_VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 1
+    MINOR = 2
     TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')

data/lib/feed_updater.rb

@@ -55,6 +55,41 @@ if !defined?(FeedTools::FEED_TOOLS_VERSION)
 end
 
 require 'benchmark'
+require 'thread'
+require 'logger'
+
+class FeedUpdaterLogger < Logger
+  attr_accessor :prefix
+  
+  alias_method :old_log, :log
+  def log(level, message)
+    if defined?(@prefix) && @prefix != nil
+      self.old_log(level, self.prefix + message)
+    else
+      self.old_log(level, message)
+    end
+  end
+
+  def debug(message)
+    self.log(0, message)
+  end
+
+  def info(message)
+    self.log(1, message)
+  end
+
+  def warn(message)
+    self.log(2, message)
+  end
+
+  def error(message)
+    self.log(3, message)
+  end
+
+  def fatal(message)
+    self.log(4, message)
+  end
+end
 
 module FeedTools
   # A simple daemon for scheduled updating of feeds.
@@ -127,7 +162,11 @@ module FeedTools
     # Returns the logger object.
     def logger()
       if !defined?(@logger) || @logger.nil?
-        @logger = Logger.new(self.log_file)
+        @logger = FeedUpdaterLogger.new(self.log_file)
+        @logger.level = self.updater_options[:log_level]
+        @logger.datetime_format = nil
+        @logger.progname = nil
+        @logger.prefix = "FeedUpdater".ljust(20)
       end
       return @logger
     end
@@ -139,7 +178,11 @@ module FeedTools
         self.logger.close()
       rescue IOError
       end
-      @logger = Logger.new(self.log_file)
+      @logger = FeedUpdaterLogger.new(self.log_file)
+      @logger.level = self.updater_options[:log_level]
+      @logger.datetime_format = nil
+      @logger.progname = nil
+      @logger.prefix = "FeedUpdater".ljust(20)
     end
     
     # Returns a list of feeds to be updated.
@@ -174,7 +217,9 @@ module FeedTools
     def updater_options()
       if !defined?(@updater_options) || @updater_options.nil?
         @updater_options = {
-          :start_delay => true
+          :start_delay => true,
+          :thread => 1,
+          :log_level => 0
         }
       end
       return @updater_options
@@ -234,6 +279,7 @@ module FeedTools
     
     # Starts the daemon.
     def start()
+      self.logger.prefix = "FeedUpdater".ljust(20)
       if !defined?(@@on_update) || @@on_update.nil?
         raise "No on_update handler block given."
       end
@@ -241,6 +287,9 @@ module FeedTools
         puts "FeedUpdater is already running."
         return self.pid
       end
+      if defined?(ActiveRecord)
+        ActiveRecord::Base.allow_concurrency = true
+      end
 
       @initial_directory = File.expand_path(".")
 
@@ -280,9 +329,13 @@ module FeedTools
               self.update_feeds()
             end
             self.logger.info(
-              "#{@feed_href_list.size} feeds updated " +
+              "#{@feed_href_list.size} feed(s) updated " +
               "in #{result.real.round} seconds.")
-            ObjectSpace.garbage_collect
+            if !defined?(@feed_href_list_override) ||
+                !@feed_href_list_override
+              @feed_href_list = nil
+            end
+            ObjectSpace.garbage_collect()
             sleepy_time = 1.hour - result.real.round
             if sleepy_time > 0
               self.logger.info(
@@ -304,11 +357,16 @@ module FeedTools
       else
         @application.start()
       end
+      self.logger.prefix = nil
+      self.logger.level = 0
       self.logger.info("-" * 79)
       self.logger.info("Daemon started, " +
         "FeedUpdater #{FeedTools::FEED_UPDATER_VERSION::STRING} / " +
         "FeedTools #{FeedTools::FEED_TOOLS_VERSION::STRING}")
       self.logger.info(" @ #{Time.now.utc.to_s}")
+      self.logger.info("-" * 79)
+      self.logger.level = self.updater_options[:log_level]
+      self.logger.prefix = "FeedUpdater".ljust(20)
       @status = :running
       return self.pid
     end
@@ -340,7 +398,10 @@ module FeedTools
         end
       rescue Exception
       end
+      self.logger.prefix = "FeedUpdater".ljust(20)
+      self.logger.level = 0
       self.logger.info("Daemon stopped.")
+      self.logger.level = self.updater_options[:log_level]
       @status = :stopped
       return nil if self.application.nil?
       begin
@@ -358,40 +419,170 @@ module FeedTools
     
     # Updates all of the feeds.
     def update_feeds()
+      self.logger.prefix = "FeedUpdater".ljust(20)
+      ObjectSpace.garbage_collect()
       if defined?(@feed_href_list_override) && @feed_href_list_override
         self.feed_href_list()
       else
+        self.logger.info("Loading default feed list...")
         @feed_href_list =
-            FeedTools.feed_cache.find(:all).collect do |cache_object|
-          cache_object.href
+          (FeedTools.feed_cache.find(:all).collect do |cache_object|
+            cache_object.href
+          end)
+      end
+      self.logger.info("Updating #{@feed_href_list.size} feed(s)...")
+      ObjectSpace.garbage_collect()
+      
+      threads = []
+      thread_slices = []
+      thread_slice_size =
+        (@feed_href_list.size / self.updater_options[:threads])
+
+      for i in 0...self.updater_options[:threads]
+        if i != self.updater_options[:threads] - 1
+          thread_slices << @feed_href_list[
+            (i * thread_slice_size)...((i + 1) * thread_slice_size)]
+        else
+          thread_slices << @feed_href_list[
+            (i * thread_slice_size)..-1]
         end
       end
-      self.logger.info("Updating #{@feed_href_list.size} feeds...")
-      for href in @feed_href_list
-        begin
-          begin
-            feed = nil
-            feed_load_benchmark = Benchmark.measure do
-              feed = FeedTools::Feed.open(href)
-            end
-            unless @@on_update.nil?
-              self.cloaker(&(@@on_update)).bind(self).call(
-                feed, feed_load_benchmark.real)
-            end
-          rescue Exception => error
-            if @@on_error != nil
-              self.cloaker(&(@@on_error)).bind(self).call(href, error)
-            else
-              self.logger.info("Error updating '#{href}':")
+      ObjectSpace.garbage_collect()
+
+      begin_updating = false
+      self.logger.info(
+        "Starting up #{self.updater_options[:threads]} thread(s)...")
+      
+      mutex = Mutex.new
+      for i in 0...self.updater_options[:threads]
+        updater_thread = Thread.new do
+          self.logger.level = self.updater_options[:log_level]
+          self.logger.datetime_format = "%s"
+          self.logger.progname = "FeedUpdater".ljust(20)
+          
+          while !Thread.current.respond_to?(:thread_id) &&
+              begin_updating == false
+            Thread.pass
+          end
+          mutex.synchronize do
+            self.logger.prefix = "Thread #{Thread.current.thread_id} ".ljust(20)
+            self.logger.info("Thread started.")
+            
+            begin
+              FeedTools.feed_cache.initialize_cache()
+              if !FeedTools.feed_cache.set_up_correctly?
+                self.logger.info("Problem with cache connection...")
+              end
+            rescue Exception => error
               self.logger.info(error)
             end
+          
+            self.logger.info(
+              "Thread ##{Thread.current.thread_id} handling " +
+              "#{thread_slices[Thread.current.thread_id].size} feeds...")
           end
-        rescue Exception => error
-          self.logger.info("Critical unhandled error.")
-          self.logger.info("Error updating '#{href}':")
-          self.logger.info(error)
+          
+          ObjectSpace.garbage_collect()
+          Thread.pass
+          href_list = thread_slices[Thread.current.thread_id]
+
+          for i in 0...href_list.size
+            href = nil
+            progress = nil
+            mutex.synchronize do
+              href = href_list[i]
+              Thread.current.progress =
+                (href_list.index(href).to_f / href_list.size.to_f) * 100
+              progress = sprintf("%.2f", Thread.current.progress)
+            end
+            begin
+              begin
+                feed = nil
+                feed_load_benchmark = Benchmark.measure do
+                  feed = FeedTools::Feed.open(href)
+                end
+                Thread.pass
+                if feed.live?
+                  unless @@on_update.nil?
+                    mutex.synchronize do
+                      progress = sprintf("%.2f", Thread.current.progress)
+                      self.logger.prefix = ("Thread #{Thread.current.thread_id} " +
+                        "(#{progress}%)"
+                        ).ljust(20)
+                      self.cloaker(&(@@on_update)).bind(self).call(
+                        feed, feed_load_benchmark.real)
+                    end
+                  end
+                else
+                  mutex.synchronize do
+                    progress = sprintf("%.2f", Thread.current.progress)
+                    self.logger.prefix = ("Thread #{Thread.current.thread_id} " +
+                      "(#{progress}%)"
+                      ).ljust(20)
+                    self.logger.info(
+                      "'#{href}' unchanged, skipping.")
+                  end
+                end
+              rescue Exception => error
+                mutex.synchronize do
+                  progress = sprintf("%.2f", Thread.current.progress)
+                  self.logger.prefix = ("Thread #{Thread.current.thread_id} " +
+                    "(#{progress}%)"
+                    ).ljust(20)
+                  if @@on_error != nil
+                    self.cloaker(&(@@on_error)).bind(self).call(href, error)
+                  else
+                    self.logger.error("Error updating '#{href}':")
+                    self.logger.error(error.class.name + ": " + error.message)
+                    self.logger.error(error.class.backtrace)
+                  end
+                end
+              end
+            rescue Exception => error
+              mutex.synchronize do
+                progress = sprintf("%.2f", Thread.current.progress)
+                self.logger.prefix = ("Thread #{Thread.current.thread_id} " +
+                  "(#{progress}%)"
+                  ).ljust(20)
+                self.logger.fatal("Critical unhandled error.")
+                self.logger.fatal("Error updating '#{href}':")
+                self.logger.fatal(error.class.name + ": " + error.message)
+                self.logger.fatal(error.class.backtrace)
+              end
+            end
+            ObjectSpace.garbage_collect()
+            Thread.pass
+          end          
+        end
+        threads << updater_thread        
+        class <<updater_thread
+          attr_accessor :thread_id
+          attr_accessor :progress
+        end
+        updater_thread.thread_id = i
+      end
+      mutex.synchronize do
+        self.logger.prefix = "FeedUpdater".ljust(20)
+        self.logger.info(
+          "#{threads.size} thread(s) successfully started...")
+        begin_updating = true
+      end
+      Thread.pass
+      
+      ObjectSpace.garbage_collect()
+      Thread.pass
+      for i in 0...threads.size
+        mutex.synchronize do
+          self.logger.prefix = "FeedUpdater".ljust(20)
+          self.logger.info(
+            "Joining on thread #{threads[i].thread_id}...")
         end
+        threads[i].join
       end
+      self.logger.prefix = "FeedUpdater".ljust(20)
+      ObjectSpace.garbage_collect()
+      
+      self.logger.progname = nil
       unless @@on_complete.nil?
         self.cloaker(&(@@on_complete)).bind(self).call(@feed_href_list)
       end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.8.11
 specification_version: 1
 name: feedupdater
 version: !ruby/object:Gem::Version 
-  version: 0.1.0
-date: 2006-04-13 00:00:00 -07:00
+  version: 0.2.0
+date: 2006-04-17 00:00:00 -07:00
 summary: Automatic feed updater daemon.
 require_paths: 
   - lib
@@ -32,7 +32,6 @@ files:
   - bin
   - CHANGELOG
   - config
-  - doc
   - example
   - lib
   - log
@@ -55,7 +54,6 @@ files:
   - lib/feed_updater/vendor/daemons/pid.rb
   - lib/feed_updater/vendor/daemons/pidfile.rb
   - lib/feed_updater/vendor/daemons/pidmem.rb
-  - config/feed_updater.pid
   - config/feed_updater.yml
   - example/custom_updater.rb
 test_files: []

data/config/feed_updater.pid

@@ -1 +0,0 @@
-617