feedupdater 0.2.1 → 0.2.2

data/CHANGELOG

@@ -1,3 +1,7 @@
+== FeedUpdater 0.2.2
+ * better distribution of work across threads
+ * fixed major threading bug
+ * added configuration variable for sleep time
 == FeedUpdater 0.2.1
  * no longer attempts to update feeds that have been updated very recently
  * fixed logging levels

data/config/feed_updater.yml

@@ -9,6 +9,9 @@ load_script: example/custom_updater.rb
 # Never set it higher than 1 if you're on a shared server.
 threads: 5
 
+# The amount of time to sleep between updates.  Expressed in minutes.
+sleep_time: 65
+
 # This is the path to store the pid files.
 pid_file_path: config
 

data/lib/feed_updater/version.rb

@@ -2,7 +2,7 @@ module FeedTools
   module FEED_UPDATER_VERSION #:nodoc:
     MAJOR = 0
     MINOR = 2
-    TINY  = 1
+    TINY  = 2
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/feed_updater.rb

@@ -226,7 +226,8 @@ module FeedTools
       if !defined?(@updater_options) || @updater_options.nil?
         @updater_options = {
           :start_delay => true,
-          :thread => 1,
+          :threads => 1,
+          :sleep_time => 60,
           :log_level => 0
         }
       end
@@ -334,7 +335,7 @@ module FeedTools
           # The main feed update loop.
           loop do
             result = nil
-            sleepy_time = 1.hour
+            sleepy_time = self.updater_options[:sleep_time].to_i.minutes
             begin
               result = Benchmark.measure do
                 self.update_feeds()
@@ -342,7 +343,9 @@ module FeedTools
               self.logger.info(
                 "#{@feed_href_list.size} feed(s) updated " +
                 "in #{result.real.round} seconds.")
-              sleepy_time = 1.hour - result.real.round
+              sleepy_time =
+                (self.updater_options[:sleep_time].to_i.minutes -
+                  result.real.round)
             rescue Exception => error
               self.logger.error("Feed update sequence errored out.")
               self.logger.error(error.class.name + ": " + error.message)
@@ -354,11 +357,14 @@ module FeedTools
             ObjectSpace.garbage_collect()
             if sleepy_time > 0
               self.logger.info(
-                "Sleeping for #{(sleepy_time / 60.0).round} minutes...")
+                "Sleeping for #{(sleepy_time / 1.minute.to_f).round} " +
+                "minutes...")
               sleep(sleepy_time)
             else
               self.logger.info(
-                "Update took more than 60 minutes, restarting immediately.")
+                "Update took more than " +
+                "#{self.updater_options[:sleep_time].to_i} minutes, " +
+                "restarting immediately.")
             end
           end
         end
@@ -432,6 +438,20 @@ module FeedTools
       self.start()
     end
     
+    def progress_precentage()
+      if !defined?(@remaining_href_list) || !defined?(@feed_href_list)
+        return nil
+      end
+      if @remaining_href_list.nil? || @feed_href_list.nil?
+        return nil
+      end
+      if @feed_href_list == []
+        return nil
+      end
+      return 100.0 - (100.0 *
+        (@remaining_href_list.size.to_f / @feed_href_list.size.to_f))
+    end
+    
     # Updates all of the feeds.
     def update_feeds()
       self.logger.level = 0
@@ -467,22 +487,10 @@ module FeedTools
       end
       self.logger.info("Updating #{@feed_href_list.size} feed(s)...")
       self.logger.level = self.updater_options[:log_level]
-      ObjectSpace.garbage_collect()
       
-      threads = []
-      thread_slices = []
-      thread_slice_size =
-        (@feed_href_list.size / self.updater_options[:threads])
+      @threads = []
+      @remaining_href_list = @feed_href_list.dup
 
-      for i in 0...self.updater_options[:threads]
-        if i != self.updater_options[:threads] - 1
-          thread_slices << @feed_href_list[
-            (i * thread_slice_size)...((i + 1) * thread_slice_size)]
-        else
-          thread_slices << @feed_href_list[
-            (i * thread_slice_size)..-1]
-        end
-      end
       ObjectSpace.garbage_collect()
 
       begin_updating = false
@@ -501,7 +509,8 @@ module FeedTools
             Thread.pass
           end
           mutex.synchronize do
-            self.logger.prefix = "Thread #{Thread.current.thread_id} ".ljust(20)
+            self.logger.prefix =
+              "Thread #{Thread.current.thread_id} ".ljust(20)
             self.logger.info("Thread started.")
             
             begin
@@ -512,23 +521,16 @@ module FeedTools
             rescue Exception => error
               self.logger.info(error)
             end
-          
-            self.logger.info(
-              "Thread ##{Thread.current.thread_id} handling " +
-              "#{thread_slices[Thread.current.thread_id].size} feeds...")
           end
           
           ObjectSpace.garbage_collect()
           Thread.pass
-          href_list = thread_slices[Thread.current.thread_id]
 
-          for i in 0...href_list.size
+          while @remaining_href_list.size > 0
             progress = nil
             mutex.synchronize do
-              Thread.current.href = href_list[i]
-              Thread.current.progress =
-                (href_list.index(Thread.current.href).to_f /
-                  href_list.size.to_f) * 100
+              Thread.current.progress = self.progress_precentage()
+              Thread.current.href = @remaining_href_list.shift()
               progress = sprintf("%.2f", Thread.current.progress)
             end
             begin
@@ -594,7 +596,7 @@ module FeedTools
             Thread.pass
           end          
         end
-        threads << updater_thread        
+        @threads << updater_thread        
         class <<updater_thread
           attr_accessor :thread_id
           attr_accessor :progress
@@ -606,20 +608,20 @@ module FeedTools
       mutex.synchronize do
         self.logger.prefix = "FeedUpdater".ljust(20)
         self.logger.info(
-          "#{threads.size} thread(s) successfully started...")
+          "#{@threads.size} thread(s) successfully started...")
         begin_updating = true
       end
       Thread.pass
       
       ObjectSpace.garbage_collect()
       Thread.pass
-      for i in 0...threads.size
+      for i in 0...@threads.size
         mutex.synchronize do
           self.logger.prefix = "FeedUpdater".ljust(20)
           self.logger.info(
-            "Joining on thread #{threads[i].thread_id}...")
+            "Joining on thread #{@threads[i].thread_id}...")
         end
-        threads[i].join
+        @threads[i].join
       end
       self.logger.prefix = "FeedUpdater".ljust(20)
       ObjectSpace.garbage_collect()

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.8.11
 specification_version: 1
 name: feedupdater
 version: !ruby/object:Gem::Version 
-  version: 0.2.1
-date: 2006-04-20 00:00:00 -07:00
+  version: 0.2.2
+date: 2006-05-17 00:00:00 -04:00
 summary: Automatic feed updater daemon.
 require_paths: 
   - lib