background_queue 0.2.0

data/spec/background_queue/server_lib/sorted_workers_spec.rb

@@ -0,0 +1,122 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue_server'
+
+
+class BasicWorker
+  attr_accessor :connections
+  def initialize(connections)
+    @connections = connections
+  end
+end
+
+describe BackgroundQueue::ServerLib::SortedWorkers do
+
+  
+  
+  context "#add_worker" do
+    it "adds the worker to the correct position in list" do
+      w1 = BasicWorker.new(1)
+      w2 = BasicWorker.new(2)
+      w3 = BasicWorker.new(3)
+      w4 = BasicWorker.new(0)
+      subject.add_worker(w1)
+      subject.add_worker(w2)
+      subject.add_worker(w3)
+      subject.add_worker(w4)
+      subject.worker_list.should eq([w4, w1, w2, w3])
+    end
+  end
+  
+  context "#remove_worker" do
+    it "just removed the worker from the array" do
+      w1 = double("worker")
+      subject.add_worker(w1)
+      subject.worker_list[0].should be(w1)
+      subject.remove_worker(w1)
+      subject.worker_list.should have(0).items
+    end
+  end
+  
+  context "#adjust_worker" do
+    it "moves the worker forward if it now has less connections to the worker before it" do
+      w1 = BasicWorker.new(1)
+      w2 = BasicWorker.new(1)
+      
+      subject.add_worker(w1)
+      subject.add_worker(w2)
+      w2.connections = 0
+      
+      subject.adjust_worker(w2)
+      subject.worker_list.should eq([w2, w1])
+    end
+    
+    it "moves the worker back if it now has more connections to the worker after it" do
+      w1 = BasicWorker.new(1)
+      w2 = BasicWorker.new(2)
+      
+      subject.add_worker(w1)
+      subject.add_worker(w2)
+      
+      w1.connections = 3
+      subject.adjust_worker(w1)
+      subject.worker_list.should eq([w2, w1])
+    end
+    
+    it "leaves the worker where it is if its positions hasnt changed" do
+      w1 = BasicWorker.new(1)
+      w2 = BasicWorker.new(2)
+      w3 = BasicWorker.new(3)
+      
+      subject.add_worker(w1)
+      subject.add_worker(w2)
+      subject.add_worker(w3)
+      subject.worker_list.should eq([w1,w2,w3])
+      
+      subject.adjust_worker(w2)
+      subject.worker_list.should eq([w1,w2,w3])
+      
+    end
+    
+    it "leaves where where it is if connection counts are the same" do
+      w1 = BasicWorker.new(1)
+      w2 = BasicWorker.new(2)
+      w3 = BasicWorker.new(3)
+      w4 = BasicWorker.new(4)
+      
+      subject.add_worker(w1)
+      subject.add_worker(w2)
+      subject.add_worker(w3)
+      subject.add_worker(w4)
+      subject.worker_list.should eq([w1,w2,w3,w4])
+      
+      w2.connections = 3
+      subject.adjust_worker(w2)
+      subject.worker_list.should eq([w1,w2,w3,w4])
+      w2.connections = 1
+      subject.adjust_worker(w2)
+      subject.worker_list.should eq([w1,w2,w3,w4])
+    end
+    
+    it "can jump multiple places" do
+      w1 = BasicWorker.new(1)
+      w2 = BasicWorker.new(2)
+      w3 = BasicWorker.new(2)
+      w4 = BasicWorker.new(2)
+      
+      subject.add_worker(w1)
+      subject.add_worker(w2)
+      subject.add_worker(w3)
+      subject.add_worker(w4)
+      subject.worker_list.should eq([w1,w4,w3,w2])
+      
+      w2.connections = 3
+      subject.adjust_worker(w2)
+      subject.worker_list.should eq([w1,w4,w3,w2])
+      w3.connections = 1
+      subject.adjust_worker(w3)
+      subject.worker_list.should eq([w1,w3,w4,w2])
+    end
+  end
+  
+  
+end

data/spec/background_queue/server_lib/task_registry_spec.rb

@@ -0,0 +1,69 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue_server'
+
+
+describe BackgroundQueue::ServerLib::TaskRegistry do
+
+  let(:task) { SimpleTask.new(:owner_id, :job_id, :task_id, 3) }
+  let(:task2) { SimpleTask.new(:owner_id, :job_id, :task_id, 3) }
+  
+  context "#register_waiting_task" do
+    it "will add task to waiting list" do
+      subject.register_waiting_task(task)
+      subject.waiting_tasks[:task_id].should eq(task)
+    end
+    
+    it "will remove and existing waiting task with same id" do
+      subject.register_waiting_task(task)
+      subject.register_waiting_task(task2)
+      subject.waiting_tasks[:task_id].should_not eq(task)
+      subject.waiting_tasks[:task_id].should eq(task2)
+    end
+  end
+  
+  context "#get_waiting_task" do
+    it "will get a matching task and remove it from the list" do
+      subject.register_waiting_task(task)
+      subject.waiting_tasks.should have(1).item
+      subject.get_waiting_task(:task_id).should eq(task)
+      subject.waiting_tasks.should have(0).items
+    end
+    
+    it "will return nil if not waiting task" do
+      subject.get_waiting_task(:task_id).should eq(nil)
+    end
+  end
+
+  context "#register" do
+    it "will add the task if it does not exist and return :new" do
+      subject.register(task).should eq([:new, nil])
+      subject.tasks.should have(1).item
+    end
+    
+    it "will add the task to waiting list if running and return :waiting" do
+      subject.register(task)
+      task.running = true
+      subject.register(task2).should eq([:waiting, nil])
+    end
+    
+    it "will replace an existing task and return the existing task" do
+      subject.register(task)
+      subject.register(task2).should eq([:existing, task])
+    end
+  end
+  
+  context "#de_register" do
+    it "will remove the task from the list of tasks and return nil if none are waiting" do
+      subject.register(task)
+      subject.de_register(task.id).should eq(nil)
+    end
+    
+    it "will remove the task from the list of tasks and return waiting task if none are waiting" do
+      subject.register(task)
+      task.running = true
+      subject.register(task2)
+      subject.de_register(task.id).should eq(task2)
+      subject.tasks[task.id].should eq(task2)
+    end
+  end
+end

data/spec/background_queue/server_lib/task_spec.rb

@@ -0,0 +1,20 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue_server'
+
+
+describe BackgroundQueue::ServerLib::Task do
+
+  subject { BackgroundQueue::ServerLib::Task.new(:owner_id, :job_id, :id, :priority, :worker, :params, {}) }
+  
+  context "#set_worker_status" do
+    it "calls the jobs set_worker_status" do
+      job = double("job")
+      status = {}
+      job.should_receive(:set_worker_status).with({:task_id=>:id, :exclude=>false}).and_return(:something)
+      subject.set_job(job)
+      subject.set_worker_status(status).should eq(:something)
+    end
+  end
+  
+  
+end

data/spec/background_queue/server_lib/thread_manager_spec.rb

@@ -0,0 +1,106 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue_server'
+
+
+describe BackgroundQueue::ServerLib::ThreadManager do
+  
+
+  let(:server) {
+    SimpleServer.new
+  }
+  
+  subject { BackgroundQueue::ServerLib::ThreadManager.new(server, 5) }
+  
+  context "#protect_access" do
+    it "will grab a mutex and serialize access" do
+      Mutex.any_instance.should_receive(:synchronize).and_yield
+      subject.protect_access {
+        5
+      }.should eq(5)
+    end
+  end
+  
+  
+  context "#control_access" do
+    it "will serialize access then execute block if concurrent number of threads is below threshold" do
+      subject.control_access {
+        5
+      }.should eq(5)
+    end
+    
+    it "will wait on condition if number of threads is above threshold" do
+      subject.max_threads = 0
+      ConditionVariable.any_instance.should_receive(:wait).and_return(nil)
+      subject.control_access {
+        5
+      }.should eq(5)
+    end
+    
+  end
+  
+  context "#signal_access" do
+    it "will signal a waiting thread if concurrent number of threads is below threshold" do
+      ConditionVariable.any_instance.should_receive(:signal).and_return(nil)
+      subject.signal_access
+    end
+    
+    it "will do nothing is number of threads is above threshold" do
+      subject.max_threads = 0
+      ConditionVariable.any_instance.should_not_receive(:signal)
+      subject.signal_access
+    end
+  end
+  
+  context "#wait_on_access" do
+    it "will wait on the condition" do
+      ConditionVariable.any_instance.should_receive(:wait).and_return(nil)
+      subject.wait_on_access
+    end
+  end
+  
+  context "#change_concurrency" do
+    it "will do nothing if the concurrency is reduced" do
+      ConditionVariable.any_instance.should_not_receive(:signal)
+      subject.change_concurrency(0)
+      subject.max_threads.should eq(0)
+    end
+    
+    it "will wake threads when concurrency is increased" do
+      ConditionVariable.any_instance.should_receive(:signal).twice
+      subject.change_concurrency(7)
+      subject.max_threads.should eq(7)
+    end
+  end
+  
+  context "#start" do
+    it "will start a number of threads and call run on a worker thread object" do
+      
+        
+      subject.max_threads = 2
+      runner = double("runner")
+      #runner.should_receive(:run).twice
+      
+      Thread.should_receive(:new).twice #.and_yield(runner)
+      
+      BackgroundQueue::ServerLib::WorkerThread.should_receive(:new).twice.and_return(runner)
+      
+      subject.start(BackgroundQueue::ServerLib::WorkerThread)
+      subject.running_threads.should eq(2)
+    end
+  end
+  
+  #hmmm.. after refactor this is tricky to test...
+  #context "#wait" do
+  #  it "will wait for each thread to finish" do
+  #    thread = double("thread")
+  #    thread.should_receive(:join).twice
+  #    Thread.should_receive(:new).twice.and_return(thread)
+  #    subject.max_threads = 2
+  #    subject.start(BackgroundQueue::ServerLib::WorkerThread)
+  #    subject.running_threads.should eq(2)
+  #    subject.wait
+  #    subject.running_threads.should eq(0)
+  #  end
+  #end
+  
+end

data/spec/background_queue/server_lib/worker_balancer_spec.rb

@@ -0,0 +1,127 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue_server'
+
+
+describe BackgroundQueue::ServerLib::WorkerBalancer do
+
+ 
+  
+  context "#initialize" do
+    it "loads the workers from the config" do
+      config = BackgroundQueue::ServerLib::Config.new([double("wc", :uri=>:worker_1), double("wc", :uri=>:worker_2)], :secret, :memcache, :address, :cons, :sopts, :task_file)
+      server = SimpleServer.new(:config=>config)
+      
+      balancer = BackgroundQueue::ServerLib::WorkerBalancer.new(server)
+      balancer.available_workers.worker_list[0].uri.should eq(:worker_1)
+      balancer.available_workers.worker_list[1].uri.should eq(:worker_2)
+    end
+  end
+  
+  context "#initialized" do
+  
+    let(:config) {
+      BackgroundQueue::ServerLib::Config.new([double("wc", :uri=>:worker_1), double("wc", :uri=>:worker_2), double("w3", :uri=>:worker_3)], :secret, :memcache, :address, :cons, :sopts, :task_file)
+    }
+    subject { BackgroundQueue::ServerLib::WorkerBalancer.new(SimpleServer.new(:config=>config)) }
+     
+     
+    context "#register_start" do
+      it "will increment the running count for the worker and reposition it" do
+        w1 = subject.available_workers.worker_list[0]
+        w1.connections.should eq(0)
+        subject.__prv__register_start(w1)
+        w1.connections.should eq(1)
+        subject.available_workers.worker_list[0].should_not be(w1)
+        subject.available_workers.worker_list.last.should be(w1)
+      end
+    end
+    
+    context "#register_finish" do
+      it "will decrement the running count for the worker and reposition it" do
+        w1 = subject.available_workers.worker_list.last
+        w1.connections.should eq(0)
+        subject.__prv__register_finish(w1)
+        w1.connections.should eq(-1)
+        subject.available_workers.worker_list.last.should_not be(w1)
+        subject.available_workers.worker_list.first.should be(w1)
+      end
+    end
+    
+    context "#register_offline" do
+      it "will remove the worker from the available workers and add it to offline workers" do
+        w1 = subject.available_workers.worker_list[0]
+        subject.__prv__register_offline(w1)
+        w1.offline?.should be_true
+        subject.available_workers.worker_list[0].should_not be(w1)
+        subject.offline_workers.first.should be(w1)
+      end
+      
+      it "will will not double add workers to offline list" do
+        w1 = subject.available_workers.worker_list[0]
+        subject.__prv__register_offline(w1)
+        subject.__prv__register_offline(w1)
+        subject.offline_workers.should have(1).item
+      end
+    end
+    
+    context "#register_online" do
+      it "will add the worker back into the available workers list" do
+        w1 = subject.available_workers.worker_list[0]
+        subject.__prv__register_offline(w1)
+        w1.offline?.should be_true
+        subject.available_workers.worker_list[0].should_not be(w1)
+        subject.offline_workers.first.should be(w1)
+        
+        subject.__prv__register_online(w1)
+        w1.offline?.should be_false
+        subject.available_workers.worker_list[0].should be(w1)
+        subject.offline_workers.should have(0).items
+      end
+    end
+    
+    context "#build_poll_task" do
+      it "creates a fake task using itself as the job" do
+        task = subject.__prv__build_poll_task
+        task.worker.should eq(:poll_worker)
+        subject.__prv__build_poll_task.should be(task)
+      end
+    end
+    
+    context "#check_offline" do
+      it "will poll the workers" do
+        w1 = subject.available_workers.worker_list[0]
+        subject.__prv__register_offline(w1)
+        BackgroundQueue::ServerLib::WorkerClient.any_instance.should_receive(:send_request).with(w1, subject.__prv__build_poll_task,  :secret ).and_return(true)
+        subject.should_receive(:register_online).with(w1)
+        subject.check_offline
+        
+      end
+    end
+    
+    context "#get_next_worker" do
+      it "will get the worker with the least number of active connections" do
+        w1 = subject.available_workers.worker_list[0]
+        subject.should_receive(:register_start).with(w1)
+        subject.get_next_worker.should be(w1)
+      end
+      
+      it "will return nil if no workers available" do
+        subject.available_workers.worker_list[0] = nil
+        subject.get_next_worker.should be_nil
+      end
+    end
+    
+    context "#finish_using_worker" do
+      it "will not register offline if online" do
+        subject.should_not_receive(:register_offline).with(:worker)
+        subject.should_receive(:register_finish).with(:worker)
+        subject.finish_using_worker(:worker, true)
+      end
+      
+      it "will register offline if not online" do
+        subject.should_receive(:register_offline).with(:worker)
+        subject.finish_using_worker(:worker, false)
+      end
+    end
+  end
+end