resque-director 1.0.1 → 1.1.0

data/Gemfile

@@ -1,7 +1,6 @@
 source "http://rubygems.org"
 
-gem "json"
-gem 'resque'
+gem 'resque', "~> 1.14.0"
 
 group :development do
   gem "rspec", "~> 2.3.0"

data/README.rdoc

@@ -6,6 +6,8 @@ Resque Director is a plugin for the Resque queueing system (http://github.com/de
 
 resque-director is mainly useful for when you are managing a large number of workers and don't want to waste resources keeping all of them waiting when they are not being used. Also useful in queues where the influx of jobs can change dramatically from time to time: enabling more workers during the times when the queue is filling up more quickly, and less in the opposite scenario. Different queues can be given different directions as well.
 
+resque-director does not operate on workers that serve multiple queues, and does not include those workers in any scaling calculations.
+
 == Usage
 
 When creating your jobs you should include (make sure to include not extend) Resque::Plugins::Director and add direction options. 
@@ -33,7 +35,7 @@ For Example:
 
 <b>wait_time</b>:: the time that it will wait after adding or removing a worker before being allowed to add or remove workers again. The default is 60 seconds.
 
-== Worker Options
+=== Worker Options
 
 <b>start_override</b>:: This will run exactly what you put in the command override as a system command to start A SINGLE worker, allowing you to customize the starting of a worker fully. The system command "QUEUE=queue_name rake resque:work" is run by default, where queue_name is whatever queue the job is running.
 
@@ -53,6 +55,16 @@ For Example:
 
 A worker will be removed if the jobs in the queue fall below half of the <b>max_queue</b>, or if the time it takes for a job to be pulled off of a queue falls below half of the <b>max_time</b>. Workers will be scaled down to the minimum if there are no jobs on the queue.
 
+=== Logging
+
+To add director logging you can pass a logger as an option. It will prepend all log messages with "DIRECTORS LOG:" and will log when scaling up or down a worker. It will also log when a worker wants to scale up or down but is unable to do so due to the fact that the maximum or minimum number of workers has been reached for that queue. 
+
+=== Logger Options 
+
+<b>logger</b>:: This will set the logger that will be used. It takes a Logger from the Ruby Standard Library. If this is not set the director will not write to any logs.
+
+<b>log_level</b>:: This sets the level to log at as a symbol. The default level is :warn.
+
 === Special Cases
 
 * If a max_worker is less than min_worker then the default for max_worker will be used (there will be no maximum). 

data/VERSION

@@ -1 +1 @@
-1.0.1
+1.1.0

data/lib/resque/plugins/director/config.rb

@@ -13,7 +13,9 @@ module Resque
           :max_queue      => 0,
           :wait_time      => 60,
           :start_override => nil,
-          :stop_override  => nil
+          :stop_override  => nil,
+          :logger         => nil,
+          :log_level      => :warn
         }
         
         def reset!
@@ -22,6 +24,10 @@ module Resque
             self.instance_variable_set("@#{key.to_s}", default)
           end
         end
+        
+        def log(message)
+          @logger.send(@log_level, "DIRECTORS LOG: #{message}") if @logger
+        end
 
         def setup(options={})
           DEFAULT_OPTIONS.each do |key, value|

data/lib/resque/plugins/director/scaler.rb

@@ -7,7 +7,7 @@ module Resque
           def scale_up(number_of_workers=1)
             number_of_workers = WorkerTracker.new.total_to_add(number_of_workers)
             scaling(number_of_workers) do
-              number_of_workers.times { start }
+              start(number_of_workers)
             end
           end
         
@@ -51,18 +51,22 @@ module Resque
             time_passed >= Config.wait_time
           end
           
-          def start
+          def start(number_of_workers)
+            Config.log("starting #{number_of_workers} workers on queue:#{Config.queue}") if number_of_workers > 0
             default_command = "QUEUE=#{Config.queue} rake resque:work &"
-            system(Config.start_override || default_command)
+            number_of_workers.times { system(Config.start_override || default_command) }
           end
           
           def stop(tracker, number_of_workers)
+            Config.log("stopping #{number_of_workers} workers on queue:#{Config.queue}") if number_of_workers > 0
             if Config.stop_override
               number_of_workers.times { system(Config.stop_override) }
             else
               valid_workers = tracker.workers.select{|w| w.hostname == `hostname`.chomp}
               worker_pids = valid_workers[0...number_of_workers].map(&:pid)
-              worker_pids.each {|pid| Process.kill("QUIT", pid)}
+              worker_pids.each do |pid| 
+                Process.kill("QUIT", pid) rescue nil
+              end
             end
           end
         end

data/lib/resque/plugins/director/worker_tracker.rb

@@ -25,12 +25,16 @@ module Resque
         def total_to_add(number_to_start)
           return number_to_start if Config.max_workers <= 0
           scale_limit = Config.max_workers - @number_working
+          Config.log("WORKER MAX REACHED: wanted to start #{number_to_start} workers on queue:#{Config.queue}") if scale_limit <= 0
           number_to_start > scale_limit ? scale_limit : number_to_start
         end
         
         def total_to_remove(number_to_stop)
           min_workers = Config.min_workers <= 0 ? 1 : Config.min_workers
           scale_limit = @number_working - min_workers
+          if scale_limit <= 0 && Config.min_workers > 0
+            Config.log("WORKER MIN REACHED: wanted to stop #{number_to_stop} workers on queue:#{Config.queue}")
+          end
           number_to_stop > scale_limit ? scale_limit : number_to_stop
         end
         

data/resque-director.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{resque-director}
-  s.version = "1.0.1"
+  s.version = "1.1.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = [%q{Nolan Frausto}]
-  s.date = %q{2011-08-20}
+  s.date = %q{2011-08-23}
   s.description = %q{resque plugin for dynamically adding/removing workers to a queue}
   s.email = %q{nrfrausto@gmail.com}
   s.extra_rdoc_files = [
@@ -50,23 +50,20 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<json>, [">= 0"])
-      s.add_runtime_dependency(%q<resque>, [">= 0"])
+      s.add_runtime_dependency(%q<resque>, ["~> 1.14.0"])
       s.add_development_dependency(%q<rspec>, ["~> 2.3.0"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
     else
-      s.add_dependency(%q<json>, [">= 0"])
-      s.add_dependency(%q<resque>, [">= 0"])
+      s.add_dependency(%q<resque>, ["~> 1.14.0"])
       s.add_dependency(%q<rspec>, ["~> 2.3.0"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
       s.add_dependency(%q<rcov>, [">= 0"])
     end
   else
-    s.add_dependency(%q<json>, [">= 0"])
-    s.add_dependency(%q<resque>, [">= 0"])
+    s.add_dependency(%q<resque>, ["~> 1.14.0"])
     s.add_dependency(%q<rspec>, ["~> 2.3.0"])
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.6.4"])

data/spec/resque/plugins/director/config_spec.rb

@@ -34,4 +34,20 @@ describe Resque::Plugins::Director::Config do
       subject.wait_time.should == 60
     end
   end
+  
+  describe "log" do
+    it "logs message to a logger using given log level if specified" do
+      log = mock('Logger')
+      subject.setup(:logger => log, :log_level => :info)
+      log.should_receive(:info).with("DIRECTORS LOG: test message")
+      subject.log("test message")
+    end
+    
+    it "defaults log level to warn" do
+      log = mock('Logger')
+      subject.setup(:logger => log)
+      log.should_receive(:warn).with("DIRECTORS LOG: test message")
+      subject.log("test message")
+    end
+  end
 end

data/spec/resque/plugins/director/scaler_spec.rb

@@ -83,17 +83,15 @@ describe Resque::Plugins::Director::Scaler do
     end
     
     it "should scale down a single worker by default" do
-      worker2 = Resque::Worker.new(:test)
-      Resque.should_receive(:workers).and_return [@worker, worker2]
+      Resque.should_receive(:workers).and_return [@worker, @worker]
       
       Process.should_receive(:kill).once
       subject.scale_down
     end
     
     it "should scale down multiple workers" do
-      worker2 = Resque::Worker.new(:test)
-      Resque.should_receive(:workers).and_return [@worker, @worker, worker2]
-      [@worker, worker2].each { |w| Process.should_receive(:kill).with("QUIT", w.pid) }
+      Resque.should_receive(:workers).and_return [@worker, @worker, @worker]
+      Process.should_receive(:kill).with("QUIT", @worker.pid)
       subject.scale_down(2)
     end
     
@@ -105,11 +103,17 @@ describe Resque::Plugins::Director::Scaler do
       subject.scale_down(2)
     end
     
+    it "should not throw exceptions when process throws exception" do
+      Resque.should_receive(:workers).and_return [@worker, @worker]
+      Process.should_receive(:kill).and_throw(:Exception)
+      lambda { subject.scale_down }.should_not raise_error
+    end
+    
     it "should not scale down workers on different queues" do
       worker2 = Resque::Worker.new(:not_test)
       @worker.stub(:pid => 1) 
       worker2.stub(:pid => 2)
-      Resque.should_receive(:workers).and_return [@worker, @worker, worker2]
+      Resque.should_receive(:workers).and_return [worker2, @worker, @worker, worker2]
       
       Process.should_not_receive(:kill).with("QUIT", worker2.pid)
       Process.should_receive(:kill).with("QUIT", @worker.pid)

data/spec/resque/plugins/director_spec.rb

@@ -120,7 +120,6 @@ describe Resque::Plugins::Director do
     end
     
     describe "with length and time" do
-      
       before do
         @start_time = {:resdirecttime => (Time.now - 10).utc.to_i}
       end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: resque-director
 version: !ruby/object:Gem::Version 
-  hash: 21
+  hash: 19
   prerelease: 
   segments: 
   - 1
-  - 0
   - 1
-  version: 1.0.1
+  - 0
+  version: 1.1.0
 platform: ruby
 authors: 
 - Nolan Frausto
@@ -15,38 +15,26 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-20 00:00:00 Z
+date: 2011-08-23 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        hash: 3
+        hash: 47
         segments: 
+        - 1
+        - 14
         - 0
-        version: "0"
+        version: 1.14.0
   version_requirements: *id001
-  name: json
-  prerelease: false
-  type: :runtime
-- !ruby/object:Gem::Dependency 
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  version_requirements: *id002
   name: resque
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency 
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -57,12 +45,12 @@ dependencies:
         - 3
         - 0
         version: 2.3.0
-  version_requirements: *id003
+  version_requirements: *id002
   name: rspec
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency 
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -73,12 +61,12 @@ dependencies:
         - 0
         - 0
         version: 1.0.0
-  version_requirements: *id004
+  version_requirements: *id003
   name: bundler
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency 
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -89,12 +77,12 @@ dependencies:
         - 6
         - 4
         version: 1.6.4
-  version_requirements: *id005
+  version_requirements: *id004
   name: jeweler
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency 
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -103,7 +91,7 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  version_requirements: *id006
+  version_requirements: *id005
   name: rcov
   prerelease: false
   type: :development