resque-sliders 0.0.10 → 0.0.11

data/README.md

@@ -47,13 +47,17 @@ Options:
     -p, --pidfile PIDFILE            PID File location
     -f, --force                      FORCE KILL ANY OTHER RUNNING KEWATCHERS
     -v, --verbose                    Verbosity (Can be specified more than once, -vv)
-    -m, --max MAX                    Max Children
+    -m, --max MAX                    Max Children (default: 10)
+    -t, --time TIME                  Total Time (in minutes) to wait for ALL Workers to die before having them force killed (default: 2 minutes)
+    -w, --wait WAIT_TIME             Minimum Time (in seconds) to wait for individual Worker to die at a time (default: 25 seconds)
     -h, --help                       This help
 ```
 
 **Important Options**
 
 * `Max Children (-m|--max MAX)`: Maximum number of workers to run on host (default: 10)
+* `Total Time (-t|--time TIME)`: How long you want to wait before sending `TERM` to resque (like `kill -9`) (default: 2 minutes)
+* `Wait Time (-w|--wait WAIT_TIME)`: How many seconds **MINIMUM** we spend in blocking wait() call per worker when cleaning up zombies (default: 25 seconds)
 * `Rakefile (-r|--rakefile RAKEFILE)`: Pass along a rakefile to use when calling `rake ... resque:work` - shouldn't be needed if run from project directory
 * `Force (-f|--force)`: Force any currently running KEWatcher processes to QUIT, waiting for it to do so, and starting in its place
 * `RAILS_ENV`: If you're using rails, you need to set your RAILS_ENV variable

data/Rakefile

@@ -1,10 +1,13 @@
-require 'bundler/gem_tasks'
-require 'resque/tasks'
+$LOAD_PATH.unshift 'helpers'
+$LOAD_PATH.unshift 'lib'
 #
 # Setup
 #
 
-$LOAD_PATH.unshift 'lib'
+
+require 'bundler/gem_tasks'
+require 'resque/tasks'
+require 'resque_job'
 
 def command?(command)
   system("type #{command} > /dev/null 2>&1")

data/bin/kewatcher

@@ -35,14 +35,22 @@ OptionParser.new do |opt|
     options[:force] = true
   end
 
-  opt.on("-v", "--verbose", "Verbosity") do
+  opt.on("-v", "--verbose", "Verbosity (Can be specified more than once, -vv)") do
     options[:verbosity] += 1
   end
 
-  opt.on("-m", "--max MAX", "Max Children") do |max|
+  opt.on("-m", "--max MAX", "Max Children (default: 10)") do |max|
     options[:max_children] = max.to_i
   end
 
+  opt.on("-t", "--time TIME", "Total Time (in minutes) to wait for ALL Workers to die before having them force killed (default: 2 minutes)") do |ttime|
+    options[:ttime] = ttime.to_f
+  end
+
+  opt.on("-w", "--wait WAIT_TIME", "Minimum Time (in seconds) to wait for individual Worker to die at a time (default: 25 seconds)") do |wait|
+    options[:wait] = wait.to_f
+  end
+
   opt.on("-h", "--help", "This help") do
     puts opt
     exit

data/lib/resque-sliders/kewatcher.rb

@@ -17,6 +17,8 @@ module Resque
         # Initialize daemon with options from command-line.
         def initialize(options={})
           @verbosity = (options[:verbosity] || 0).to_i
+          @ttime = options[:ttime] || 2
+          @zombie_wait = options[:wait] || 30
           @hostile_takeover = options[:force]
           @rakefile = File.expand_path(options[:rakefile]) rescue nil
           @rakefile = File.exists?(@rakefile) ? @rakefile : nil if @rakefile
@@ -29,7 +31,7 @@ module Resque
           @pids = Hash.new # init pids array to track running children
           @need_queues = Array.new # keep track of pids that are needed
           @dead_queues = Array.new # keep track of pids that are dead
-          @zombie_pids = Array.new # keep track of zombie's we kill and dont watch()
+          @zombie_pids = Hash.new # keep track of zombie's we kill and dont watch(), with elapsed time we've waited for it to die
 
           Resque.redis = case options[:config]
             when Hash
@@ -74,7 +76,7 @@ module Resque
             procline if tick
 
             sleep(interval) # microsleep
-            kill_zombies! # need to cleanup ones we've killed
+            kill_zombies! unless shutdown? # need to cleanup ones we've killed
 
             @pids.keys.each do |pid|
               begin
@@ -289,14 +291,36 @@ module Resque
         end
 
         def kill_zombies!
-          @zombie_pids.dup.each do |pid|
+          return if @zombie_pids.empty?
+          zombies = @zombie_pids.dup
+          wait = 60 * @ttime / zombies.length # in seconds
+          mark = Time.now # start of loop
+          zombies.each do |pid,elapsed|
+            elapsed += Time.now - mark
+            mark = Time.now # mark that we updated elapsed; time becomes relative
+            time_left = (@ttime * 60.0) - elapsed # seconds
+            wait = time_left < wait ? time_left : wait
+            wait = @zombie_wait if wait < @zombie_wait && wait > 0
+            wait = 0.1 if wait <= 0
+            log! "Waiting for Zombie: #{pid} (#{'%.2f' % wait} seconds) => #{'%.2f' % elapsed} elapsed"
             begin
-              log! "Waiting for Zombie: #{pid}"
               # Issue wait() to make sure pid isn't forgotten
-              Process.wait(@zombie_pids.pop)
-            rescue Errno::ECHILD
+              Timeout::timeout(wait) { Process.wait(pid) }
+            rescue Timeout::Error
+              elapsed += Time.now - mark
+              mark = Time.now # mark that we updated elapsed
+              log! "TIMEOUT waiting for zombie #{pid} => #{'%.2f' % elapsed} elapsed"
+              (log "Waited more than #{@ttime} minutes for #{pid}. Force quitting..."; Process.kill('TERM',pid)) if elapsed / 60.0 >= @ttime
               next
+            rescue Errno::ECHILD # child is gone
+            ensure
+              elapsed += Time.now - mark
+              mark = Time.now # mark that we updated elapsed
+              log! "Elapsed incr: #{elapsed}"
+              zombies.each_key { |x| zombies[x] = elapsed } # reset all to current elapsed
+              @zombie_pids = zombies # make sure to update root Hash
             end
+            @zombie_pids.delete(pid)
           end
         end
 
@@ -308,7 +332,7 @@ module Resque
             log! "Child #{pid} already dead, sad day. (#{@pids.keys.length-1}) #{e}"
           ensure
             # Keep track of ones we issued QUIT to
-            @zombie_pids << pid
+            @zombie_pids[pid] = 0 # set to 0 wait time
           end
         end
 

data/lib/resque-sliders/version.rb

@@ -1,7 +1,7 @@
 module Resque
   module Plugins
     module ResqueSliders
-      Version = VERSION = '0.0.10'
+      Version = VERSION = '0.0.11'
     end
   end
 end

metadata

@@ -1,48 +1,40 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: resque-sliders
-version: !ruby/object:Gem::Version 
-  hash: 11
+version: !ruby/object:Gem::Version
+  version: 0.0.11
   prerelease: 
-  segments: 
-  - 0
-  - 0
-  - 10
-  version: 0.0.10
 platform: ruby
-authors: 
+authors:
 - Kevin Mullin
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-02-08 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2012-02-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: resque
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &2156319820 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 43
-        segments: 
-        - 1
-        - 15
-        - 0
+      - !ruby/object:Gem::Version
         version: 1.15.0
   type: :runtime
-  version_requirements: *id001
-description: "    Resque-Sliders is a plugin for Resque that enables you to control multiple hosts'\n    running resque workers with a monitor PID watching over them.  From the resque-web UI, you\n    can add/delete/change which queues are running on each host running the monitor PID.\n    Sliders are in the UI and allow you to adjust how many workers on either host should be running.\n"
+  prerelease: false
+  version_requirements: *2156319820
+description: ! "    Resque-Sliders is a plugin for Resque that enables you to control
+  multiple hosts'\n    running resque workers with a monitor PID watching over them.
+  \ From the resque-web UI, you\n    can add/delete/change which queues are running
+  on each host running the monitor PID.\n    Sliders are in the UI and allow you to
+  adjust how many workers on either host should be running.\n"
 email: kevin@kpmullin.com
-executables: 
+executables:
 - kewatcher
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - README.md
 - MIT-LICENSE
-files: 
+files:
 - Gemfile
 - MIT-LICENSE
 - README.md
@@ -78,38 +70,30 @@ files:
 - test/test_helper.rb
 homepage: https://github.com/kmullin/resque-sliders
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
 rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
-summary: "Resque-Sliders: a plugin for resque that controls which resque workers are running on each host, from within Resque-web"
-test_files: 
+summary: ! 'Resque-Sliders: a plugin for resque that controls which resque workers
+  are running on each host, from within Resque-web'
+test_files:
 - test/redis-test.conf
 - test/resque-sliders_test.rb
 - test/test_helper.rb