bluth 0.7.0 → 0.7.2

data/CHANGES.txt

@@ -1,5 +1,25 @@
 BLUTH, CHANGES
 
+#### 0.7.2 (2011-04-10) ###############################
+
+* FIXED: Bluth::TimingBelt.notch now caches notches by absolute time (not just HH:MM)
+* CHANGE: Gracefully rescue Errno::ECONNREFUSED and try to reconnect
+* ADDED: Bluth::Worker.reconnect!
+* ADDED: Bluth::Worker.onerror
+
+
+#### 0.7.1 (2011-03-06) ###############################
+
+* FIXED: replace-worker command now always grabs the oldest worker
+  and gracefully exits when there's no workers.
+* CHANGE: Bluth::Gob objects have a ttl of 2 hours (up from 1)
+* CHANGE: Bluth::Worker.interval is now 1 sec (down from 2)
+* CHANGE: Default queuetimeout is now 30 seconds (down from 30)
+* ADDED: Bluth::Worker processes now handle USR1 and USR2 signals
+  for enabling/disabling debugging and increasing/decreasing
+  Bluth.queuetimeout (respectively)
+
+
 #### 0.7.0 (2011-03-04) ###############################
 
 * ADDED: backtrace field to Bluth::Gob

data/Rakefile

@@ -25,7 +25,7 @@ begin
     gem.email = "delano@solutious.com"
     gem.homepage = "http://github.com/delano/bluth"
     gem.authors = ["Delano Mandelbaum"]
-    gem.add_dependency("familia",    ">= 0.6.5")
+    gem.add_dependency("familia",    ">= 0.7.1")
     gem.add_dependency('sysinfo',    '>= 0.7.3')
     gem.add_dependency('annoy')
     

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :MAJOR: 0
 :MINOR: 7
-:PATCH: 0
+:PATCH: 2

data/bin/bluth

@@ -66,6 +66,7 @@ class Bluth::CLI::Definition
   
   #TODO: command :enqueue
   
+  #option :o, :ontheminute, "Start the worker precisely on the minute"
   command :start_worker => Bluth::CLI
   command :start_scheduler => Bluth::CLI
   
@@ -77,6 +78,7 @@ class Bluth::CLI::Definition
   command :stop_scheduler => Bluth::CLI
   
   about "Stop the oldest worker and start a new instance in its place."
+  #option :o, :ontheminute, "Start the worker precisely on the minute"
   command :replace_worker => Bluth::CLI
   
   command :workers => Bluth::CLI  

data/bluth.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{bluth}
-  s.version = "0.7.0"
+  s.version = "0.7.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Delano Mandelbaum"]
-  s.date = %q{2011-03-04}
+  s.date = %q{2011-04-12}
   s.default_executable = %q{bluth}
   s.description = %q{A Redis queuing system built on top of Familia}
   s.email = %q{delano@solutious.com}
@@ -50,16 +50,16 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<familia>, [">= 0.6.5"])
+      s.add_runtime_dependency(%q<familia>, [">= 0.7.1"])
       s.add_runtime_dependency(%q<sysinfo>, [">= 0.7.3"])
       s.add_runtime_dependency(%q<annoy>, [">= 0"])
     else
-      s.add_dependency(%q<familia>, [">= 0.6.5"])
+      s.add_dependency(%q<familia>, [">= 0.7.1"])
       s.add_dependency(%q<sysinfo>, [">= 0.7.3"])
       s.add_dependency(%q<annoy>, [">= 0"])
     end
   else
-    s.add_dependency(%q<familia>, [">= 0.6.5"])
+    s.add_dependency(%q<familia>, [">= 0.7.1"])
     s.add_dependency(%q<sysinfo>, [">= 0.7.3"])
     s.add_dependency(%q<annoy>, [">= 0"])
   end

data/lib/bluth.rb

@@ -35,7 +35,7 @@ module Bluth
   class Shutdown < Familia::Problem; end
   @db = 0
   @env = :dev
-  @queuetimeout = 60 #.seconds
+  @queuetimeout = 30 #.seconds
   @handlers = []
   @locks = []
   @sysinfo = nil
@@ -66,6 +66,18 @@ module Bluth
     def connect
       instance_eval &onconnect unless onconnect.nil?
     end
+    def reconnect! delay=5
+      sleep delay
+      success = begin
+        Familia.reconnect_all!
+        (Bluth::Gob.redis.ping.to_s.upcase == 'PONG')
+      rescue => ex
+        Familia.info "  #{ex.message}"
+        false
+      end
+      Familia.info "  reconnected: #{success}"
+      success
+    end      
   end
   
   def Bluth.clear_locks
@@ -173,6 +185,10 @@ module Bluth
         gob.current_queue = :running
         gob.save
       end
+      
+    rescue Errno::ECONNREFUSED => ex
+      raise ex
+      
     rescue => ex
       if queue.nil?
         Familia.info "ERROR: #{ex.message}"
@@ -255,7 +271,7 @@ module Bluth
     MAX_ATTEMPTS = 3.freeze unless defined?(Gob::MAX_ATTEMPTS)
     include Familia
     prefix [:bluth, :gob]
-    ttl 3600 #.seconds
+    ttl 60*60*2 # 2 hours
     index :jobid
     field :jobid => Gibbler::Digest
     field :handler => String

data/lib/bluth/cli.rb

@@ -29,7 +29,7 @@ module Bluth
     def start_worker worker_class=Bluth::Worker
       if @global.daemon
         worker = worker_class.new
-        Familia.info "Created: #{worker.rediskey}"
+        Familia.info "Created: #{worker.index}"
         worker.daemonize
         worker.run
       else
@@ -56,14 +56,21 @@ module Bluth
     
     def replace_worker worker_class=Bluth::Worker
       Bluth.connect
-      @global.daemon = true
-      worker = worker_class.instances.first  # grabs the oldest worker
-      kill_worker worker, worker_class
-      start_worker worker_class
+      workers = worker_class.instances.members.select { |w| w.host == Bluth.sysinfo.hostname }
+      if workers.empty?
+        Familia.info "No workers running on #{Bluth.sysinfo.hostname}"
+      else
+        workers.sort! { |a,b| a.created <=> b.created }
+        oldest_worker = workers.first
+        Familia.info "Replacing #{oldest_worker.index}/#{oldest_worker.pid} (running since #{Time.at(oldest_worker.created).utc})"
+        kill_worker oldest_worker, worker_class
+        @global.daemon = true
+        start_worker worker_class
+      end
     end
     
     def workers worker_class=Bluth::Worker
-      Familia.info worker_class.all.collect &:rediskey
+      Familia.info worker_class.all.collect &:index
     end
     
     private 
@@ -73,7 +80,7 @@ module Bluth
         Familia.info "No such worker"
         exit 1
       else
-        Familia.info "Killing #{worker.rediskey}"
+        Familia.info "Killing #{worker.index}"
         worker.kill @option.force
       end
     end

data/lib/bluth/timingbelt.rb

@@ -65,18 +65,19 @@ module Bluth
         (time + (mins*60)).strftime('%H:%M')
       end
       def notch mins=0, filter=nil, time=now
+        cache_key = [now(mins, time).to_i, filter].join(':')
         key = rediskey(stamp(mins, time), filter)
         @notchcache ||= {}
-        if @notchcache[key].nil?
-          @notchcache[key] ||= Familia::Set.new key, 
+        if @notchcache[cache_key].nil?
+          @notchcache[cache_key] ||= Familia::Set.new key, 
               :ttl => 2*60*60, # 2 hours
               :extend => Bluth::TimingBelt::Notch, 
               :db => Bluth::TimingBelt.db
-          @notchcache[key].stamp = stamp(mins, time) 
-          @notchcache[key].filter = filter
-          @notchcache[key].time = now(mins, time)
+          @notchcache[cache_key].stamp = stamp(mins, time) 
+          @notchcache[cache_key].filter = filter
+          @notchcache[cache_key].time = now(mins, time)
         end
-        @notchcache[key]
+        @notchcache[cache_key]
       end
       def priority minutes=2, filter=nil, time=now
         (0..minutes).to_a.reverse.collect { |min| notch(min*-1, filter, time) }

data/lib/bluth/worker.rb

@@ -78,15 +78,27 @@ module Bluth
         me = new nil, nil, wid
         super(me.index)
       end
+      def reconnect! name=nil
+        Familia.info "Reconnecting #{name}"
+        attempt = 0
+        success = false
+        reconnect_tries.times {
+          attempt += 1
+          Familia.info " reconnecting in #{reconnect_delay} (\##{attempt} of #{reconnect_tries})..."
+          success = Bluth.reconnect! reconnect_delay
+          break if success
+        }
+        success
+      end
       def run!(*args)
         me = new
-        Familia.info "Created: #{me.rediskey}"
+        Familia.info "Created: #{me.index}"
         me.run!
         me
       end
       def run(*args)
         me = new
-        Familia.info "Created: #{me.rediskey}"
+        Familia.info "Created: #{me.index}"
         me.run
         me
       end
@@ -99,6 +111,10 @@ module Bluth
         @onstart = blk unless blk.nil?
         @onstart
       end
+      def onerror &blk
+        @onerror = blk unless blk.nil?
+        @onerror
+      end
       def onexit &blk
         @onexit = blk unless blk.nil?
         @onexit
@@ -113,9 +129,11 @@ module Bluth
   end
   
   class Worker < Storable
-    @interval = 2 #.seconds
+    @interval = 1 #.seconds
+    @reconnect_delay = 15 #.seconds
+    @reconnect_tries = 20
     class << self
-      attr_accessor :interval
+      attr_accessor :interval, :reconnect_delay, :reconnect_tries
     end
     include WorkerBase
     include Familia
@@ -143,17 +161,41 @@ module Bluth
       end
     end
     
+    def carefully
+      begin
+        yield
+      rescue Errno::ECONNREFUSED => ex
+        
+        unless Bluth::Worker.reconnect! self.index
+          self.class.onerror.call ex, self if self.class.onerror 
+          Familia.info "Reconnect failed :["
+        end
+      
+      rescue => ex
+        if self.class.onerror 
+          self.class.onerror.call ex, self
+        else
+          Familia.info ex.message
+          Familia.ld ex.backtrace
+          problem!
+        end
+        #if problem > 5
+        #  ## TODO: SEND EMAIL
+        #  task.unschedule unless task.nil? # Kill this worker b/c something is clearly wrong
+        #  destroy!
+        #  EM.stop
+        #  exit 1
+        #end
+      end
+    end
+    
     def run!
       begin
         Bluth.connect
         self.class.runblock :onstart
-        find_gob
-      rescue => ex
-        msg = "#{ex.class}: #{ex.message}"
-        Familia.info msg
-        Familia.trace :EXCEPTION, msg, caller[1] if Familia.debug?
-        self.class.runblock :onexit
-        destroy!
+        carefully do
+          find_gob
+        end
       rescue Interrupt => ex
         puts $/, "Exiting..."
         self.class.runblock :onexit
@@ -164,25 +206,39 @@ module Bluth
     def run
       begin
         @process_id = $$
-        scheduler = Rufus::Scheduler.start_new
+        @scheduler = Rufus::Scheduler.start_new
         Bluth.connect
         self.class.runblock :onstart
         Familia.info "Setting interval: #{Worker.interval} sec (queuetimeout: #{Bluth.queuetimeout})"
         Familia.reconnect_all! # Need to reconnect after daemonize
         save
-        scheduler.every Worker.interval, :blocking => true do |task|
-          Familia.ld "#{$$} TICK @ #{Time.now.utc}" if Familia.debug?
-          sleep rand
-          find_gob task
+        Signal.trap("USR1") do
+          Familia.debug = (Familia.debug == false)
+          Familia.info "Debugging is #{Familia.debug ? 'enabled' : 'disabled'}"
         end
-        scheduler.join
-            
-      rescue => ex
-        msg = "#{ex.class}: #{ex.message}"
-        Familia.info msg
-        Familia.trace :EXCEPTION, msg, caller[1] if Familia.debug?
-        self.class.runblock :onexit
-        destroy!
+        @usr2_reduce = true
+        Signal.trap("USR2") do
+          @usr2_reduce = false if Bluth.queuetimeout <= 2
+          @usr2_reduce = true if Bluth.queuetimeout >= 60
+          if @usr2_reduce
+            #Worker.interval /= 2.0
+            Bluth.queuetimeout /= 2
+          else
+            #Worker.interval *= 2.0
+            Bluth.queuetimeout *= 2
+          end
+          Familia.info "Set intervals: #{Worker.interval} sec / #{Bluth.queuetimeout} sec"
+        end
+        ## TODO: on_the_minute = Time.at(BS.quantize(Stella.now, 1.minute)+1.minute).utc  ## first_at
+        ## @option.ontheminute
+        @task = @scheduler.every Worker.interval, :blocking => true, :first_in => '2s' do |task|
+          carefully do
+            Familia.ld "#{$$} TICK @ #{Time.now.utc}" if Familia.debug?
+            find_gob task
+          end
+        end
+        @scheduler.join 
+        
       rescue Interrupt => ex
         puts <<-EOS.gsub(/(?:^|\n)\s*/, "\n")
           Exiting...
@@ -191,13 +247,13 @@ module Bluth
         EOS
         # We reconnect to the queue in case we're currently
         # waiting on a brpop (blocking pop) timeout.
-        self.class.runblock :onexit
         destroy!
+      ensure
+        self.class.runblock :onexit
       end
       
     end
     
-    
     private 
     
     # DO NOT call return from this method
@@ -222,7 +278,7 @@ module Bluth
             job.success!
             self.success!
           end
-        end   
+        end
       rescue Bluth::Shutdown => ex
         msg = "Shutdown requested: #{ex.message}"
         job.success! msg
@@ -239,18 +295,11 @@ module Bluth
         job.failure! ex.message
         self.failure!
       rescue => ex
-        Familia.info ex.message
         Familia.info ex.backtrace
         job.retry! "#{ex.class}: #{ex.message}" if job
-        problem!
-        #if problem > 5
-        #  ## TODO: SEND EMAIL
-        #  task.unschedule unless task.nil? # Kill this worker b/c something is clearly wrong
-        #  destroy!
-        #  EM.stop
-        #  exit 1
-        #end
+        raise ex
       end
+        
     end
   end
   

data/try/30_timingbelt_try.rb

@@ -9,11 +9,11 @@ Bluth::TimingBelt.redis.flushdb
 
 ## Knows now
 Bluth::TimingBelt.now(0, @now).to_s
-#=> '2011-02-14 00:00:00 UTC'
+#=> RUBY_VERSION <= '1.9' ? 'Mon Feb 14 00:00:00 UTC 2011' : '2011-02-14 00:00:00 UTC'
 
 ## Now can have an offset
 Bluth::TimingBelt.now(5, @now).to_s
-#=> '2011-02-14 00:05:00 UTC'
+#=> RUBY_VERSION <= '1.9' ? 'Mon Feb 14 00:05:00 UTC 2011' : '2011-02-14 00:05:00 UTC'
 
 ## Can create a timestamp
 Bluth::TimingBelt.stamp 0, @now
@@ -94,7 +94,6 @@ notches.first.name unless notches.first.nil?
 ## Can calculate the difference between two notches
 notch1 = Bluth::TimingBelt.notch
 notch2 = Bluth::TimingBelt.notch 67
-puts notch2.name
 notch2 - notch1
 #=> 67
 

metadata

@@ -2,7 +2,7 @@
 name: bluth
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.7.0
+  version: 0.7.2
 platform: ruby
 authors: 
 - Delano Mandelbaum
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-04 00:00:00 -05:00
+date: 2011-04-12 00:00:00 -04:00
 default_executable: bluth
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -21,7 +21,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 0.6.5
+        version: 0.7.1
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency