background_queue 0.2.0

data/spec/background_queue/worker/environment_spec.rb

@@ -0,0 +1,67 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue_worker'
+
+
+describe BackgroundQueue::Worker::Environment do
+
+  let(:logger) {
+    l = double("logger")
+    l.stub(:debug)
+    l.stub(:info)
+    l.stub(:error)
+    l.stub(:warn)
+    l
+  }
+  
+  let(:params) {
+    {
+      :auth=>"ABCD",
+      :task=>{:params=>{:a=>"b"}, :owner_id=>'oid', :job_id=>'jid', :id=>'tid', :priority=>1}.to_json,
+      :server_port=>213
+    }
+  }
+  
+  let(:response) {
+    double("resp")
+  }
+  
+  let(:request) {
+    double("req", :remote_ip=>"192.168.1.1")
+  }
+  
+  let(:controller) {
+    double("controller", :logger=>logger, :response=>response, :params=>params, :request=>request)
+  }
+  
+  
+  context "#init_params" do
+    it "will load task data from params" do
+      subject.init_params(params)
+      subject.params[:a].should eq("b")
+      subject.owner_id.should eq("oid")
+      subject.job_id.should eq("jid")
+      subject.task_id.should eq("tid")
+      subject.priority.should eq(1)
+    end
+    
+    it "will error if the task definition is not valid json" do
+      params[:task] = '[123'
+      expect { subject.init_params(params) }.to raise_exception
+    end
+    
+    it "will error if the task definition is not a json hash" do
+      params[:task] = '[123]'
+      expect { subject.init_params(params) }.to raise_exception("Invalid json root object (should be hash)")
+    end
+  end
+  
+  context "#init_server_address" do
+    it "will get the server ip from the request" do
+      subject.init_server_address(controller)
+      subject.server_address.port.should eq(213)
+      subject.server_address.host.should eq("192.168.1.1")
+    end
+  end
+  
+  
+end

data/spec/background_queue/worker/worker_loader_spec.rb

@@ -0,0 +1,103 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue_worker'
+
+class ThisWorkerExists
+  
+end
+
+describe BackgroundQueue::Worker::WorkerLoader do
+
+  let(:worker_entry) {
+    BackgroundQueue::Worker::WorkerLoader::WorkerEntry.new(:worker, :path, Time.at(10), :worker_name)
+  }
+    
+  context "#worker_class_name" do
+    it "will convert to camelcase" do
+      subject.worker_class_name("something").should eq("Something")
+      subject.worker_class_name("something_else").should eq("SomethingElse")
+    end
+  end
+  
+  
+  context "#worker_path" do
+    it "will get the worker path using the worker name" do
+      BackgroundQueue::Worker::Config.worker_path = "ROOT"
+      subject.worker_path("something").should eq("ROOT/something.rb")
+      subject.worker_path("something_else").should eq("ROOT/something_else.rb")
+    end
+  end
+  
+  context "#load_worker" do
+    it "will wrap a worker in a worker entry if the file loads and defines the worker class" do
+      subject.should_receive(:load_file).with(:path).and_return(true)
+      subject.should_receive(:load_class).with(:worker_name, :path).and_return(:clazz)
+      subject.should_receive(:worker_path).with(:worker_name).and_return(:path)
+      File.should_receive(:mtime).with(:path).and_return(:datestamp)
+      worker_entry = subject.load_worker(:worker_name)
+      worker_entry.worker.should eq(:clazz)
+      worker_entry.name.should eq(:worker_name)
+      worker_entry.datestamp.should eq(:datestamp)
+      worker_entry.path.should eq(:path)
+    end
+  end
+  
+  context "#load_file" do
+    it "will load a file if it is a valid ruby file" do
+      path = File.expand_path(File.dirname(__FILE__) + '/../../resources/example_worker.rb')
+      subject.load_file(path)
+    end
+    
+    it "will raise an error if the file does not exist" do
+      path = File.expand_path(File.dirname(__FILE__) + '/../../resources/example_worker_not_found.rb')
+      expect { subject.load_file(path) }.to raise_exception
+    end
+    
+    it "will raise an error if the file is not valid ruby" do
+      path = File.expand_path(File.dirname(__FILE__) + '/../../resources/example_worker_with_error.rb')
+      expect { subject.load_file(path) }.to raise_exception
+    end
+  end
+  
+  context "#load_class" do
+    it "will create an instance of the worker class" do
+      subject.load_class("this_worker_exists", "path").class.name.should eq("ThisWorkerExists")
+    end
+    
+    it "will error if the class is not found" do
+      expect { subject.load_class("this_worker_does_not_exist", "path") }.to raise_exception("path did not define ThisWorkerDoesNotExist")
+    end
+  end
+  
+  context "#reload_if_updated" do
+    it "will reload the worker if the filestamp has changed" do
+      File.should_receive(:mtime).with(:path).and_return(Time.at(11))
+      subject.should_receive(:load_file).with(:path).and_return(true)
+      subject.should_receive(:load_class).with(:worker_name).and_return(:new_worker)
+      
+      subject.reload_if_updated(worker_entry)
+      worker_entry.worker.should eq(:new_worker)
+    end
+    
+    it "will not reload the worker if the filestamp has not changed" do
+      File.should_receive(:mtime).with(:path).and_return(Time.at(10))
+      subject.reload_if_updated(worker_entry)
+      worker_entry.worker.should eq(:worker)
+    end
+  end
+  
+  context "#get_worker" do
+    it "will load the worker if not in cache" do
+      subject.should_receive(:load_worker).with(:worker_name).and_return(worker_entry)
+      subject.get_worker(:worker_name).should eq(:worker)
+    end
+    
+    it "will cache the results" do
+      subject.should_receive(:load_worker).with(:worker_name).and_return(worker_entry)
+      subject.should_receive(:reload_if_updated).with(worker_entry).and_return(nil)
+      subject.get_worker(:worker_name).should eq(:worker)
+      subject.get_worker(:worker_name).should eq(:worker)
+    end
+  end
+  
+
+end

data/spec/background_queue_spec.rb

@@ -0,0 +1,7 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "BackgroundQueue" do
+  xit "does nothing yet" do
+
+  end
+end

data/spec/resources/config-client.yml

@@ -0,0 +1,7 @@
+development:
+  server: 
+    host: 127.0.0.1
+    port: 3000
+  memcache: 127.0.0.1:9999
+
+

data/spec/resources/config-serialize.yml

@@ -0,0 +1,12 @@
+development:
+  address: 
+    host: 127.0.0.1
+    port: 3000
+  workers: 
+  - http://127.0.0.1:8001/background_queue
+  secret: this_is_used_to_make_sure_it_is_secure
+  connections_per_worker: 10
+  system_task_options:
+    domain: the_default_domain
+  task_file: /tmp/bq_tasks
+

data/spec/resources/config.yml

@@ -0,0 +1,12 @@
+development:
+  address: 
+    host: 127.0.0.1
+    port: 3000
+  workers: 
+  - http://127.0.0.1:8001/background_queue
+  secret: this_is_used_to_make_sure_it_is_secure
+  connections_per_worker: 10
+  system_task_options:
+    domain: the_default_domain
+
+

data/spec/resources/example_worker.rb

@@ -0,0 +1,4 @@
+class ExampleWorker
+
+
+end

data/spec/resources/example_worker_with_error.rb

@@ -0,0 +1,4 @@
+classx ExampleWorker
+
+
+end

data/spec/resources/test_worker.rb

@@ -0,0 +1,8 @@
+class TestWorker < BackgroundQueue::Worker::Base
+
+  def run
+    set_progress("Not Yet..", 50)
+    set_progress("Not Yet..", 75)
+    set_progress("Done", 100)
+  end
+end

data/spec/shared/queue_registry_shared.rb

@@ -0,0 +1,216 @@
+require 'background_queue_server'
+
+shared_examples "a queue registry" do
+
+  context "queue registry management" do
+    
+    subject { new_instance }
+    
+    it "will create an queue" do
+      subject.should_receive(:create_queue).with(:queue_id) { :the_queue}
+      subject.__prv__get_queue(:queue_id, true).should eq([false, :the_queue])
+    end
+    
+    it "will find existing queue" do
+      subject.should_receive(:create_queue).with(:queue_id) { :the_queue}
+      subject.__prv__get_queue(:queue_id, true)
+      subject.stub(:create_queue).with(:queue_id) { :this_should_not_be_used }
+      subject.__prv__get_queue(:queue_id, true).should eq([true, :the_queue])
+    end
+    
+  end
+  
+  
+  context "queue items" do
+    let(:queue_class) { described_class.queue_class }
+    
+    let(:task1) { SimpleTask.new(:owner_id, :job_id, :task_id, 3) }
+    let(:task2) { SimpleTask.new(:owner_id2, :job_id, :task_id2, 2) }
+    
+    before do
+      queue_class.any_instance.stub(:add_item) do |item|
+        @priority = item.priority if @priority.nil? || @priority > item.priority
+      end
+      
+      queue_class.any_instance.stub(:priority) do 
+        @priority
+      end
+      
+      queue_class.any_instance.stub(:set_priority) do |priority|
+        @priority = priority
+      end
+      
+      queue_class.any_instance.stub(:next_item) do
+        @priority += 1 #just assume we lower priority...
+      end
+    end
+      
+    context "#add_item" do
+
+      context "new queue" do
+        
+        subject { new_instance }
+        
+        it "tracks if the queue priority changed" do
+          in_queue, queue = subject.__prv__get_queue(:queue_id, true)
+          subject.__prv__track_priority_when_adding_to_queue(queue, task1).should eq([true, nil])
+        end
+        
+        it "will add with correct priority" do
+
+          subject.add_item(task1)
+          subject.__prv__get_queues.should have(1).items
+          subject.__prv__get_queues.first.priority.should eq(3)
+        end
+      end
+      
+      context "existing queue" do
+      
+        subject { 
+          bg = new_instance
+          bg.add_item(task1)
+          bg
+        }
+        
+        it "will re-prioritise existing queue to higher priority" do
+          subject.__prv__get_queues.should have(1).items
+          subject.__prv__get_queues.first.priority.should eq(3)
+          subject.add_item(SimpleTask.new(:owner_id, :job_id, :task_id2, 2))
+          subject.__prv__get_queues.should have(1).items
+          subject.__prv__get_queues.first.priority.should eq(2)
+        end
+        
+        it "will not re-prioritise existing owner to lower priority" do
+          subject.__prv__get_queues.should have(1).items
+          subject.__prv__get_queues.first.priority.should eq(3)
+          subject.add_item(SimpleTask.new(:owner_id, :job_id, :task_id2, 4))
+          subject.__prv__get_queues.should have(1).items
+          subject.__prv__get_queues.first.priority.should eq(3)
+        end
+      end
+    end
+    
+    context "#next_item" do
+  
+      subject { 
+        bg = new_instance
+        bg.stub(:get_queue_id_from_item) { |item| item.owner_id} 
+        bg.add_item(task1)
+        bg.add_item(task2)
+        bg
+      }
+        
+      it "will add to stalled items if no items left for queue" do
+        subject.should_receive(:remove_item_from_queue).with(any_args) { [true, 2, :task]}
+        queue_class.any_instance.stub(:empty?) { true }
+        subject.should_receive(:stall_queue)
+        subject.__prv__get_queue_for_priority(2, false)
+        subject.next_item.should eq(:task)
+        subject.__prv__get_queues.should have(1).items
+        subject.__prv__get_queues.first.priority.should eq(3)
+        #make sure the owner was removed
+        subject.__prv__get_queue(:owner_id2, true).first.should eq(false)
+      end
+      
+      
+      
+      it "will lower priority when items left are lower priority" do
+        subject.should_receive(:remove_item_from_queue).with(any_args) { [true, 2, :task]}
+        queue_class.any_instance.stub(:empty?) { false }
+        
+        subject.__prv__get_queue(:owner_id2, true).last.set_priority(3)
+        
+        subject.next_item
+        subject.__prv__get_queues.should have(1).items
+        subject.__prv__get_queues.first.priority.should eq(3)
+        subject.__prv__get_queues.first.should have(2).items
+      end
+      
+      
+    end
+    
+    context "#remove_item_from_queue" do
+      
+      subject { 
+        bg = new_instance
+        bg.stub(:get_queue_id_from_item) { |item| item.owner_id} 
+        bg.add_item(task1)
+        bg.add_item(task2)
+        bg
+      }
+      
+      it "tracks if priority changed when removing next item" do
+        owner = double("owner")
+        owner.stub(:priority) { @priority ||= 2 }
+        owner.stub(:next_item) { @priority += 1; :task }
+        
+        subject.__prv__remove_item_from_queue(owner, :next).should eq([true, 2, :task])
+      end
+      
+      it "tracks if priority unchanged when removing next item" do
+        owner = double("owner")
+        owner.stub(:priority) { @priority ||= 2 }
+        owner.stub(:next_item) {:task }
+        
+        subject.__prv__remove_item_from_queue(owner, :next).should eq([false, 2, :task])
+      end
+      
+      it "tracks if priority changed when removing specific item" do
+        owner = double("owner")
+        owner.stub(:priority) { @priority ||= 2 }
+        owner.stub(:remove_item) { @priority += 1; :task }
+        
+        subject.__prv__remove_item_from_queue(owner, :task).should eq([true, 2, :task])
+      end
+      
+      it "tracks if priority unchanged when removing specific item" do
+        owner = double("owner")
+        owner.stub(:priority) { @priority ||= 2 }
+        owner.stub(:remove_item) {:task }
+        
+        subject.__prv__remove_item_from_queue(owner, :task).should eq([false, 2, :task])
+      end
+      
+    end
+    
+    
+    context "#remove_item" do
+
+      subject { 
+        bg = new_instance
+        bg.stub(:get_queue_id_from_item) { |item| item.owner_id} 
+        bg.add_item(task1)
+        bg.add_item(task2)
+        bg
+      }
+        
+      it "will remove the queue if no items left for queue" do
+        subject.should_receive(:remove_item_from_queue).with(any_args) { [true, 3, :task]}
+        queue_class.any_instance.stub(:empty?) { true }
+           
+        subject.remove_item(task1)
+        subject.__prv__get_queues.should have(1).items
+        subject.__prv__get_queues.first.priority.should eq(2)
+        #make sure the owner was removed
+        subject.__prv__get_queue(:owner_id, false).first.should be_false
+      end
+
+      it "will lower priority when items left are lower priority" do
+        
+        subject.add_item(SimpleTask.new(:owner_id2, :job_id, :task_id3, 3))
+        
+        subject.should_receive(:remove_item_from_queue).with(any_args) { [true, 2, :task]}
+        queue_class.any_instance.stub(:empty?) { false }
+        
+        subject.__prv__get_queue(:owner_id2, true).last.set_priority(3) #need to manually set the priority to 3 because we are mocking remove_item_from_queue
+        
+        subject.remove_item(task2)
+        subject.__prv__get_queues.should have(1).items #both owners should be at same priority (3)
+        subject.__prv__get_queues.first.priority.should eq(3)
+        subject.__prv__get_queues.first.should have(2).items
+      end
+    end
+  end
+  
+  
+end