resque-director 1.0.0

data/spec/resque/plugins/director/scaler_spec.rb

@@ -0,0 +1,246 @@
+require 'spec_helper'
+
+describe Resque::Plugins::Director::Scaler do
+  subject { Resque::Plugins::Director::Scaler }
+  
+  before do
+    Resque::Plugins::Director::Config.queue = "test"
+  end
+  
+  describe "#scale_up" do
+    it "should start a worker on a specific queue" do
+      subject.should_receive(:system).with("QUEUE=test rake resque:work &")
+      subject.scale_up
+    end
+    
+    it "should start the specified number of workers on a specific queue" do
+      subject.should_receive(:system).twice.with("QUEUE=test rake resque:work &")
+      subject.scale_up(2)
+    end
+    
+    it "should not start more workers than the maximum allowed" do
+      Resque::Plugins::Director::Config.setup :max_workers => 1
+      subject.should_receive(:system).once.with("QUEUE=test rake resque:work &")
+      subject.scale_up(2)
+    end
+    
+    it "should override the entire comand" do
+      Resque::Plugins::Director::Config.setup(:start_override => "run this")
+      subject.should_receive(:system).with("run this")
+      subject.scale_up
+    end
+  end
+  
+  describe "#scaling" do
+    before do
+      @times_scaled = 0
+    end
+    
+    it "should not scale workers if last time scaled is too soon" do
+      Resque::Plugins::Director::Config.setup(:wait_time => 60)
+      2.times { subject.scaling { @times_scaled += 1 } }
+      @times_scaled.should == 1
+    end
+    
+    it "should scale workers if wait time has passed" do
+      Resque::Plugins::Director::Config.setup(:wait_time => 0)
+      2.times { subject.scaling { @times_scaled += 1 } }
+      @times_scaled.should == 2
+    end
+  end
+  
+  describe "#scale_down_to_minimum" do
+    before do
+      @worker = Resque::Worker.new(:test)
+      Resque::Plugins::Director::Config.setup :min_workers => 1
+    end
+    
+    it "should scale workers down to the minimum" do
+      Resque.should_receive(:workers).and_return [@worker, @worker, @worker]
+      Process.should_receive(:kill).twice
+      subject.scale_down_to_minimum
+    end
+    
+    it "should not scale if the workers are already at the minimum" do
+      Resque.should_receive(:workers).and_return [@worker]
+      Process.should_not_receive(:kill)
+      subject.scale_down_to_minimum
+    end
+    
+    it "forces scaling by ignoring wait_time" do
+      Resque::Plugins::Director::Config.setup(:wait_time => 60, :min_workers => 2)
+      subject.scaling {}
+      Resque.should_receive(:workers).and_return [@worker, @worker, @worker]
+      Process.should_receive(:kill)
+      subject.scale_down_to_minimum
+    end
+  end
+  
+  describe "#scale_down" do
+    before do
+      @worker = Resque::Worker.new(:test)
+      Resque::Plugins::Director::Config.setup :min_workers => 0
+    end
+    
+    it "should scale down a single worker by default" do
+      worker2 = Resque::Worker.new(:test)
+      Resque.should_receive(:workers).and_return [@worker, worker2]
+      
+      Process.should_receive(:kill).once
+      subject.scale_down
+    end
+    
+    it "should scale down multiple workers" do
+      worker2 = Resque::Worker.new(:test)
+      Resque.should_receive(:workers).and_return [@worker, @worker, worker2]
+      [@worker, worker2].each { |w| Process.should_receive(:kill).with("QUIT", w.pid) }
+      subject.scale_down(2)
+    end
+    
+    it "should not scale down more than the minimum allowed workers" do
+      Resque::Plugins::Director::Config.setup :min_workers => 1
+      
+      Resque.should_receive(:workers).and_return [@worker, @worker]
+      Process.should_receive(:kill).once
+      subject.scale_down(2)
+    end
+    
+    it "should not scale down workers on different queues" do
+      worker2 = Resque::Worker.new(:not_test)
+      @worker.stub(:pid => 1) 
+      worker2.stub(:pid => 2)
+      Resque.should_receive(:workers).and_return [@worker, @worker, worker2]
+      
+      Process.should_not_receive(:kill).with("QUIT", worker2.pid)
+      Process.should_receive(:kill).with("QUIT", @worker.pid)
+      subject.scale_down
+    end
+  end
+  
+  describe "#stop" do
+    before do
+      @worker = Resque::Worker.new(:test)
+      Resque::Plugins::Director::Config.setup :min_workers => 0
+    end
+    
+    it "should kill worker by sending the QUIT signal to the workers pid" do
+      Resque.should_receive(:workers).and_return [@worker]
+      tracker = Resque::Plugins::Director::WorkerTracker.new
+      
+      Process.should_receive(:kill).with("QUIT", @worker.pid)
+      subject.send(:stop, tracker, 1)
+    end
+    
+    it "should use the stop command to stop a worker if set" do
+      Resque::Plugins::Director::Config.setup :stop_override => "run this", :min_workers => 0
+      Resque.should_receive(:workers).and_return [@worker]
+      tracker = Resque::Plugins::Director::WorkerTracker.new
+      
+      subject.should_receive(:system).with("run this")
+      Process.should_not_receive(:kill)
+      subject.send(:stop, tracker, 1)
+    end
+    
+    it "does not stop workers already set to be shutdown" do
+      @worker.should_receive(:shutdown?).and_return(true)
+      Resque.should_receive(:workers).and_return [@worker]
+      tracker = Resque::Plugins::Director::WorkerTracker.new
+      
+      Process.should_not_receive(:kill).with("QUIT", @worker.pid)
+      subject.send(:stop, tracker, 1)
+    end
+    
+    it "does not kill worker processes on different machines" do
+      @worker.stub!(:hostname => "different_machine")
+      Resque.should_receive(:workers).and_return [@worker]
+      tracker = Resque::Plugins::Director::WorkerTracker.new
+      
+      Process.should_not_receive(:kill).with("QUIT", @worker.pid)
+      subject.send(:stop, tracker, 1)
+    end
+    
+    it "stops workers on the same host if possible" do
+      @worker.stub!(:hostname => "different_machine")
+      worker2 = Resque::Worker.new(:test)
+      Resque.should_receive(:workers).and_return [@worker, worker2]
+      tracker = Resque::Plugins::Director::WorkerTracker.new
+      
+      Process.should_receive(:kill).with("QUIT", worker2.pid)
+      subject.send(:stop, tracker, 1)
+    end
+    
+    it "ignores hostname if using custom stop script" do
+      Resque::Plugins::Director::Config.setup :stop_override => "run this", :min_workers => 0
+      @worker.stub!(:hostname => "different_machine")
+      Resque.should_receive(:workers).and_return [@worker]
+      tracker = Resque::Plugins::Director::WorkerTracker.new
+      
+      subject.should_receive(:system).with("run this")
+      Process.should_not_receive(:kill)
+      subject.send(:stop, tracker, 1)
+    end
+  end
+  
+  describe "#scale_within_requirements" do
+    it "should not scale up workers if the minumum number or greater are already running" do
+      Resque::Worker.new(:test).register_worker
+      subject.should_not_receive(:scale_up)
+      subject.scale_within_requirements
+    end
+    
+    it "should scale up the minimum number of workers if non are running" do
+      Resque::Plugins::Director::Config.setup :min_workers => 2
+      subject.should_receive(:scale_up).with(2)
+      subject.scale_within_requirements
+    end
+    
+    it "should ensure at least one worker is running if min_workers is less than zero" do
+      Resque::Plugins::Director::Config.setup :min_workers => -10
+      subject.should_receive(:scale_up).with(1)
+      subject.scale_within_requirements
+    end
+    
+    it "should ensure at least one worker is running if min_workers is zero" do
+      Resque::Plugins::Director::Config.setup :min_workers => 0
+      subject.should_receive(:scale_up).with(1)
+      subject.scale_within_requirements
+    end
+    
+    it "should scale up the minimum number of workers if less than the minimum are running" do
+      Resque::Plugins::Director::Config.setup :min_workers => 2
+      Resque::Worker.new(:test).register_worker
+      subject.should_receive(:scale_up).with(1)
+      subject.scale_within_requirements
+    end
+    
+    it "should scale down the max number of workers if more than max" do
+      Resque::Plugins::Director::Config.setup :max_workers => 1
+      workers = 2.times.map { Resque::Worker.new(:test) }
+      Resque.should_receive(:workers).and_return(workers)
+      
+      subject.should_receive(:scale_down).with(1)
+      subject.scale_within_requirements
+    end
+    
+    it "should not scale down if max_workers is zero" do
+      Resque::Plugins::Director::Config.setup :max_workers => 0
+      workers = 1.times.map { Resque::Worker.new(:test) }
+      Resque.should_receive(:workers).and_return(workers)
+      
+      subject.should_not_receive(:scale_down)
+      subject.scale_within_requirements
+    end
+    
+    it "should ignore workers from other queues" do
+      Resque::Worker.new(:other).register_worker
+      subject.should_receive(:scale_up).with(1)
+      subject.scale_within_requirements
+    end
+    
+    it "should ignore workers on multiple queues" do
+      Resque::Worker.new(:test, :other).register_worker
+      subject.should_receive(:scale_up).with(1)
+      subject.scale_within_requirements
+    end
+  end
+end

data/spec/resque/plugins/director/worker_tracker_spec.rb

@@ -0,0 +1,137 @@
+require 'spec_helper'
+
+describe Resque::Plugins::Director::WorkerTracker do
+  subject { Resque::Plugins::Director::WorkerTracker }
+  
+  before do
+    Resque::Plugins::Director::Config.queue = "test"
+    @worker = Resque::Worker.new(:test)
+  end
+  
+  describe "#workers" do
+    before do
+      Resque::Plugins::Director::Config.queue = "worker_test"
+    end
+    
+    it"should return the workers for the queue" do
+      Resque::Worker.new(:other).register_worker
+      expected_worker = Resque::Worker.new(:worker_test)
+      expected_worker.register_worker
+      
+      subject.new.workers.should == [expected_worker]
+    end
+    
+    it "should not return workers working on multiple queues" do
+      Resque::Worker.new(:worker_test, :other).register_worker
+      subject.new.workers.should be_empty
+    end
+  end
+  
+  describe "#total_to_remove" do
+    it "should limit the workers to be removed to not go below minimum allowed workers" do
+      Resque.should_receive(:workers).and_return [@worker, @worker]
+      Resque::Plugins::Director::Config.setup :min_workers => 1
+      subject.new.total_to_remove(2).should == 1
+    end
+  
+    it "always keeps at least one worker" do
+      Resque.should_receive(:workers).and_return [@worker, @worker, @worker]
+      Resque::Plugins::Director::Config.setup :min_workers => 0
+      subject.new.total_to_remove(2).should == 2
+    end
+    
+    it "should return zero if there is only one worker working" do
+      Resque.should_receive(:workers).and_return [@worker]
+      Resque::Plugins::Director::Config.setup :min_workers => 0
+      subject.new.total_to_remove(1).should == 0
+    end
+  end
+
+
+  describe "#total_to_add" do
+    before do
+      Resque.should_receive(:workers).and_return [@worker, @worker]
+    end
+    
+    it "should limit the workers to be removed to not go below minimum allowed workers" do
+      Resque::Plugins::Director::Config.setup :max_workers => 3
+      subject.new.total_to_add(2).should == 1
+    end
+  
+    it "should allow the workers to be removed if it stays above the minimum" do
+      Resque::Plugins::Director::Config.setup :max_workers => 4
+      subject.new.total_to_add(2).should == 2
+    end
+  
+    it "should not limit workers added if max_workers is zero" do
+      Resque::Plugins::Director::Config.setup :max_workers => 0
+      subject.new.total_to_add(200).should == 200
+    end
+  end
+
+  describe "#total_for_requirements" do
+    it "should return the number of workers needed to meet the minimum requirement" do
+      Resque.should_receive(:workers).and_return [@worker, @worker]
+      Resque::Plugins::Director::Config.setup :min_workers => 4
+      subject.new.total_for_requirements.should == 2
+    end
+    
+    it "should return the number of workers needed to meet the maximum requirement" do
+      Resque.should_receive(:workers).and_return [@worker, @worker]
+      Resque::Plugins::Director::Config.setup :max_workers => 1
+      subject.new.total_for_requirements.should == -1
+    end
+    
+    it "should return 1 no workers are running and the minimum is zero" do
+      Resque.should_receive(:workers).and_return []
+      Resque::Plugins::Director::Config.setup :min_workers => 0
+      subject.new.total_for_requirements.should == 1
+    end
+    
+    it "should return 0 if number of workers is within requirements" do
+      Resque.should_receive(:workers).and_return [@worker, @worker]
+      Resque::Plugins::Director::Config.setup :min_workers => 1, :max_workers => 3
+      subject.new.total_for_requirements.should == 0
+    end
+  end
+  
+  describe "#total_to_go_to_minimum" do
+    it "should return the number to scale down to go to the minimum" do
+      Resque.should_receive(:workers).and_return [@worker, @worker, @worker]
+      Resque::Plugins::Director::Config.setup :min_workers => 2
+      subject.new.total_to_go_to_minimum.should == 1
+    end
+    
+    it "should return zero if already at the minimum" do
+      Resque.should_receive(:workers).and_return [@worker, @worker, @worker]
+      Resque::Plugins::Director::Config.setup :min_workers => 3
+      subject.new.total_to_go_to_minimum.should == 0
+    end
+    
+    it "should return zero if below the minimum" do
+      Resque.should_receive(:workers).and_return [@worker, @worker, @worker]
+      Resque::Plugins::Director::Config.setup :min_workers => 4
+      subject.new.total_to_go_to_minimum.should == 0
+    end
+  end
+  
+  describe "#initialize" do
+    it "sets the workers from the queue" do
+      Resque.should_receive(:workers).and_return [@worker, @worker, @worker]
+      subject.new.workers.should == [@worker, @worker, @worker]
+    end
+    
+    it "does not set workers from different queues" do
+      other_worker = Resque::Worker.new(:other)
+      Resque.should_receive(:workers).and_return [@worker, other_worker, @worker]
+      subject.new.workers.should == [@worker, @worker]
+    end
+    
+    it "does not set workers that are scheduled for shutdow" do
+      shutdown_worker = Resque::Worker.new(:test)
+      shutdown_worker.shutdown
+      Resque.should_receive(:workers).and_return [shutdown_worker, @worker, @worker]
+      subject.new.workers.should == [@worker, @worker]
+    end
+  end
+end

data/spec/resque/plugins/director_spec.rb

@@ -0,0 +1,200 @@
+require 'spec_helper'
+
+describe Resque::Plugins::Director do
+  before do
+    TestJob.direct
+  end
+  
+  it "should be a valid resque plugin" do
+    Resque::Plugin.lint(Resque::Plugins::Director)
+  end
+  
+  describe "#after_enqueue_scale_workers" do
+    it "should scale the workers to within the requirments specified" do
+      Resque::Worker.new(:test).register_worker
+      Resque::Plugins::Director::Scaler.should_receive(:scale_within_requirements)
+      Resque.enqueue(TestJob)
+    end
+    
+    it "should set the queue" do
+      Resque.enqueue(TestJob)
+      Resque::Plugins::Director::Config.queue.should == "test"
+    end
+  end
+  
+  describe "#after_perform_direct_workers" do
+    it "should scale down to the minumum workers if there are no jobs in the queue" do
+      Resque::Plugins::Director::Scaler.should_receive(:scale_down_to_minimum)
+      TestJob.after_perform_direct_workers
+    end
+    
+    it "should not scale down if there are jobs in the queue" do
+      Resque.enqueue(TestJob)
+      Resque::Plugins::Director::Scaler.should_not_receive(:scale_down_to_minimum)
+      TestJob.after_perform_direct_workers
+    end
+  end
+  
+  describe "#on_failure_direct_workers" do
+    it "should scale down to the minumum workers if there are no jobs in the queue" do
+      Resque::Plugins::Director::Scaler.should_receive(:scale_down_to_minimum)
+      TestJob.on_failure_direct_workers
+    end
+    
+    it "should not scale down if there are jobs in the queue" do
+      Resque.enqueue(TestJob)
+      Resque::Plugins::Director::Scaler.should_not_receive(:scale_down_to_minimum)
+      TestJob.on_failure_direct_workers
+    end
+  end
+  
+  describe "#before_perform_direct_workers" do
+    describe "with time" do
+      before do
+        @start_time = {:resdirecttime => (Time.now - 10).utc.to_i}
+      end
+      
+      it "should set the queue if not set" do
+        TestJob.direct :max_time => 20
+        Resque::Plugins::Director::Config.queue = nil
+        TestJob.before_perform_direct_workers(@start_time)
+        Resque::Plugins::Director::Config.queue.should == "test"
+      end
+    
+      it "should not start workers if max_time is not set" do
+        Resque::Plugins::Director::Scaler.should_not_receive(:scale_up)
+        TestJob.before_perform_direct_workers(@start_time)
+      end 
+    
+      it "should not start a worker if the time since it took is less than max_time" do
+        TestJob.direct :max_time => 20
+      
+        Resque::Plugins::Director::Scaler.should_not_receive(:scale_up)
+        TestJob.before_perform_direct_workers(@start_time)
+      end
+    
+      it "should add a worker if the time it takes the job to go through the queue is too long" do
+        TestJob.direct :max_time => 5
+        Resque::Plugins::Director::Scaler.should_receive(:scale_up)
+      
+        TestJob.before_perform_direct_workers(@start_time)
+      end
+      
+      it "should remove a worker if the queue time is below half the max" do
+        TestJob.direct :max_time => 25
+        
+        Resque::Plugins::Director::Scaler.should_receive(:scale_down)
+        TestJob.before_perform_direct_workers(@start_time)
+      end
+    end
+    
+    describe "with queue length" do
+      it "should not start workers if max_queue is not set" do
+        Resque::Plugins::Director::Scaler.should_not_receive(:scale_up)
+        TestJob.before_perform_direct_workers
+      end
+      
+      it "should not start worker if the queue length is less than max_queue" do
+        TestJob.direct :max_queue => 2
+        Resque.enqueue(TestJob)
+      
+        Resque::Plugins::Director::Scaler.should_not_receive(:scale_up)
+        TestJob.before_perform_direct_workers
+      end
+      
+      it "should start worker if the queue length is greater than max_queue" do
+        TestJob.direct :max_queue => 1
+        2.times { Resque.enqueue(TestJob) }
+      
+        Resque::Plugins::Director::Scaler.should_receive(:scale_up)
+        TestJob.before_perform_direct_workers
+      end
+      
+      it "should remove a worker if the queue length is below half the max" do
+        TestJob.direct :max_queue => 4
+        1.times { Resque.enqueue(TestJob) }
+        
+        Resque::Plugins::Director::Scaler.should_receive(:scale_down)
+        TestJob.before_perform_direct_workers
+      end
+    end
+    
+    describe "with length and time" do
+      
+      before do
+        @start_time = {:resdirecttime => (Time.now - 10).utc.to_i}
+      end
+      
+      it "should add worker if only time constraint fails" do
+        TestJob.direct :max_time => 5, :max_queue => 2
+        Resque.enqueue(TestJob)
+        Resque::Plugins::Director::Scaler.should_receive(:scale_up)
+      
+        TestJob.before_perform_direct_workers(@start_time)
+      end
+      
+      it "should add worker if only queue length constraint fails" do
+        TestJob.direct :max_time => 15, :max_queue => 1
+        2.times { Resque.enqueue(TestJob) }
+      
+        Resque::Plugins::Director::Scaler.should_receive(:scale_up)
+        TestJob.before_perform_direct_workers
+      end
+      
+      it "should not scale down if a worker is being scaled up due to time" do
+        TestJob.direct :max_queue => 4, :max_time => 5
+        1.times { Resque.enqueue(TestJob) }
+
+        Resque::Plugins::Director::Scaler.should_receive(:scale_up)
+        Resque::Plugins::Director::Scaler.should_not_receive(:scale_down)
+        TestJob.before_perform_direct_workers(@start_time)
+      end
+      
+      it "should not scale down if a worker is being scaled up due to queue" do
+        TestJob.direct :max_queue => 1, :max_time => 30
+        2.times { Resque.enqueue(TestJob) }
+
+        Resque::Plugins::Director::Scaler.should_receive(:scale_up)
+        Resque::Plugins::Director::Scaler.should_not_receive(:scale_down)
+        TestJob.before_perform_direct_workers
+      end
+      
+      it "should not scale if only one limit is met" do
+        TestJob.direct :max_queue => 3, :max_time => 15
+        1.times { Resque.enqueue(TestJob) }
+
+        Resque::Plugins::Director::Scaler.should_not_receive(:scale_up)
+        Resque::Plugins::Director::Scaler.should_not_receive(:scale_down)
+        TestJob.before_perform_direct_workers(@start_time)
+      end
+      
+      it "should not scale if no configuration options are set" do
+        Resque::Plugins::Director::Scaler.should_not_receive(:scale_up)
+        Resque::Plugins::Director::Scaler.should_not_receive(:scale_down)
+        TestJob.before_perform_direct_workers
+      end
+    end
+    
+    describe "crazy meta custom_perform" do      
+      it "should strip out the timestamp from the args before calling the original perform" do
+        TestJob.should_receive(:original_perform).with("arg")
+        TestJob.perform("arg", {:resdirecttime => 1234})
+      end
+      
+      it "should strip out timestamp if it is not a symbol" do
+        TestJob.should_receive(:original_perform).with("arg")
+        TestJob.perform("arg", {'resdirecttime' => 1234})
+      end
+      
+      it "should not strip out any args if timestamp does not exist" do
+        TestJob.should_receive(:original_perform).with("arg", {:test => 123})
+        TestJob.perform("arg", {:test => 123})
+      end
+      
+      it "should  not strip out any args if timestamp does not exist" do
+        TestJob.should_receive(:original_perform).with("arg")
+        TestJob.perform("arg")
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,45 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require 'resque'
+require 'resque-director'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+#
+# make sure we can run redis
+#
+if !system("which redis-server")
+  puts '', "** can't find `redis-server` in your path"
+  puts "** add redis-server to your PATH and try again"
+  abort ''
+end
+
+dir = File.dirname(__FILE__)
+#
+# start our own redis when the tests start,
+# kill it when they end
+#
+
+at_exit do
+  pid = `ps -e -o pid,command | grep [r]edis-test`.split(" ")[0]
+  puts "Killing test redis server [#{pid}]..."
+  `rm -f #{dir}/dump.rdb`
+  Process.kill("KILL", pid.to_i)
+end
+
+puts "Starting redis for testing at localhost:9736..."
+`redis-server #{dir}/redis-test.conf`
+Resque.redis = 'localhost:9736'
+ENV['VERBOSE'] = 'true'
+
+RSpec.configure do |config|
+  config.before(:each) do
+    Resque::Plugins::Director::Config.reset!
+    Resque::Plugins::Director::Scaler.stub!(:system) 
+    Process.stub!(:kill)
+    Resque.redis.flushall
+  end
+end

data/spec/support/test_job.rb

@@ -0,0 +1,7 @@
+class TestJob
+  include Resque::Plugins::Director
+  @queue = :test
+
+  def self.perform
+  end
+end