background_queue 0.3.0 → 0.6.0

data/lib/background_queue/worker/progress.rb

@@ -0,0 +1,152 @@
+module BackgroundQueue::Worker
+  #a way of notifying worker progress
+  class Progress
+    
+    attr_reader :registered_tasks
+  
+    def initialize(callback_object)
+      if callback_object.kind_of?(BackgroundQueue::Worker::Base)
+        @worker = callback_object
+      elsif callback_object.kind_of?(Proc) 
+        @callback = callback_object
+      else
+        @callback_object = callback_object
+      end  
+      
+      @main_caption = nil
+      @sub_caption = nil
+      
+      @finished_sub_progress = 0    
+      @current_sub_progress_size = 0
+      @current_sub_progress = 0
+      @sub_task_step_size = 0
+    end
+    
+    def start(main_caption=nil)
+      @main_caption = main_caption
+ 
+      @sub_caption = nil
+      
+      @finished_sub_progress = 0    
+      @current_sub_progress_size = 0
+      @current_sub_progress = 0
+
+      update_callback
+    end
+    
+    def set_main_caption(caption)
+      @main_caption = caption
+      update_callback
+    end
+    
+    def finish(main_caption = nil)
+      @main_caption = main_caption unless main_caption.nil?
+      @sub_caption = nil unless main_caption.nil?
+      @current_sub_progress = 0
+      @finished_sub_progress = 100
+      update_callback
+    end
+    
+    
+    def register_task(key, units)
+      @registered_tasks ||= {}
+      @registered_task_total ||= 0
+      @registered_tasks[key] = units
+      @registered_task_total += units
+    end
+    
+    def start_task(key, caption=nil)
+      if @current_sub_progress_size != 0
+        @finished_sub_progress += @current_sub_progress_size
+        @current_sub_progress_size = 0
+      end
+      @current_sub_progress_size = get_task_size(key)
+      @current_sub_progress = 0.0
+      @sub_task_step_size = 0
+      @sub_caption = caption
+      update_callback
+    end
+    
+    def get_task_size(key)
+      raise "No registered sub task (#{key})" if @registered_tasks[key].nil?
+      @registered_tasks[key].to_f / @registered_task_total * 100.0
+    end
+    
+    def set_task_steps(step_count)
+      if step_count == 0
+        @sub_task_step_size = 100
+      else
+        @sub_task_step_size = 100.0 / step_count.to_f * 100.0
+      end
+    end
+    
+    def inc
+      @current_sub_progress += @sub_task_step_size 
+      update_callback
+    end
+    
+    def set_task_progress(percent)
+      @current_sub_progress += percent
+      @current_sub_progress = 100 if @current_sub_progress > 100
+      update_callback
+    end   
+    
+    def set_task_caption(caption)
+      @sub_caption = caption
+      update_callback
+    end
+    
+    def add_note(notice)
+      update_callback_meta(:notice, notice)
+    end
+    
+    def add_warning(warning)
+      update_callback_meta(:warning, warning)
+    end
+    
+    def add_error(error)
+      update_callback_meta(:error, error)
+    end
+    
+    def set_meta_data(key, data)
+      update_callback_meta(:meta, {key=>data})
+    end
+    
+    def get_caption
+      if @main_caption && @sub_caption 
+        "#{@main_caption}: #{@sub_caption }"
+      elsif @main_caption
+        @main_caption
+      elsif @sub_caption 
+        @sub_caption 
+      else
+        ""
+      end
+    end
+    
+    def get_percent
+      @finished_sub_progress + (@current_sub_progress_size  / 100.0 * @current_sub_progress / 100.0)
+    end
+    
+    def update_callback
+      if @worker
+        @worker.set_progress(get_caption, get_percent)
+      elsif @callback
+        @callback.call(:progress, get_caption, get_percent)
+      elsif @callback_object
+        @callback_object.set_progress(get_caption, get_percent, self)
+      end
+    end
+    
+    def update_callback_meta(key, value)
+      if @worker
+        @worker.add_progress_meta(key, value)
+      elsif @callback
+        @callback.call(:meta, key, value)
+      elsif @callback_object
+        @callback_object.add_progress_meta(key, value, self)
+      end
+    end
+    
+  end
+end

data/lib/background_queue/worker/worker_loader.rb

@@ -56,7 +56,7 @@ module BackgroundQueue::Worker
       ds = File.mtime(worker_entry.path)
       if ds != worker_entry.datestamp
         load_file(worker_entry.path)
-        worker_entry.worker = load_class(worker_entry.name)
+        worker_entry.worker = load_class(worker_entry.name, worker_entry.path)
         worker_entry.datestamp = ds
       end
     end

data/lib/background_queue_worker.rb

@@ -3,3 +3,5 @@ require "background_queue/worker/environment"
 require "background_queue/worker/worker_loader"
 require "background_queue/worker/calling"
 require "background_queue/worker/config"
+require "background_queue/worker/progress"
+require "background_queue/worker/logger"

data/spec/background_queue/client_lib/connection_spec.rb

@@ -9,6 +9,10 @@ describe "Connection" do
     svr
   }
   
+  let(:socket) {
+    double("socket")
+  }
+  
   
 
   context "connecting" do
@@ -16,7 +20,8 @@ describe "Connection" do
     subject { BackgroundQueue::ClientLib::Connection.new(:client, server) }
     
     it "can successfully connect" do
-      TCPSocket.should_receive(:open).with(:host, :port) { :socket }
+      socket.should_receive(:setsockopt)
+      TCPSocket.should_receive(:open).with(:host, :port) { socket }
       subject.__prv__connect.should eq(true)
     end
     
@@ -38,6 +43,7 @@ describe "Connection" do
     subject { 
       s = BackgroundQueue::ClientLib::Connection.new(:client, server) 
       TCPSocket.should_receive(:open).with(:host, :port) { socket }
+      socket.should_receive(:setsockopt)
       s.__prv__connect
       s
     }

data/spec/background_queue/client_spec.rb

@@ -72,7 +72,8 @@ describe "Client" do
       end
       
       it "can build and send an add tasks command" do
-        subject.add_tasks(:worker, :owner_id, :job_id, :tasks, 1, {}, {} ).should eq([true, :server])
+        job_handle = subject.add_tasks(:worker, :owner_id, :job_id, :tasks, 1, {}, {} )
+        job_handle.job_id.should eq(:job_id)
       end
       
       

data/spec/background_queue/config_spec.rb

@@ -57,38 +57,26 @@ describe "Config" do
         end
       end
       
-      context "extracting the environment entry" do
-        it "gets_the_current environment from env" do
-          ENV.should_receive(:has_key?).with("RAILS_ENV") { true }
-          ENV.should_receive(:[]).with("RAILS_ENV") { :the_env }
-          BackgroundQueue::Config.__prv__current_environment.should eq(:the_env)
+    
+      context "with development environment" do
+        before do
+          BackgroundQueue::Utils.stub(:current_environment) { 'development' }
         end
         
-        it "gets_the_current environment from Rails" do
-          ENV.should_receive(:has_key?).with("RAILS_ENV") { false }
-          Rails.should_receive(:env) { :the_env }
-          BackgroundQueue::Config.__prv__current_environment.should eq(:the_env)
+        it "extracts the correct environment entry from the hash" do
+          BackgroundQueue::Config.__prv__extract_enviroment_entry({:development=>:test}, :path_that_exists).should eq(:test)
         end
         
-        context "with development environment" do
-          before do
-            BackgroundQueue::Config.stub(:current_environment) { 'development' }
-          end
-          
-          it "extracts the correct environment entry from the hash" do
-            BackgroundQueue::Config.__prv__extract_enviroment_entry({:development=>:test}, :path_that_exists).should eq(:test)
-          end
-          
-          it "errors if the YAML does not define environment entry" do
-            File.stub(:expand_path) { :expanded_path }
-            expect { BackgroundQueue::Config.__prv__extract_enviroment_entry({:test=>:test}, :path_that_exists).should eq(:test)}.to raise_error(BackgroundQueue::LoadError, "Error loading YAML for background_queue configuration file at 'expanded_path': missing enviroment root entry: development")
-          end
+        it "errors if the YAML does not define environment entry" do
+          File.stub(:expand_path) { :expanded_path }
+          expect { BackgroundQueue::Config.__prv__extract_enviroment_entry({:test=>:test}, :path_that_exists).should eq(:test)}.to raise_error(BackgroundQueue::LoadError, "Error loading YAML for background_queue configuration file at 'expanded_path': missing enviroment root entry: development")
         end
       end
+
       
       context "with extracted environment entry" do
         before do
-          BackgroundQueue::Config.stub(:current_environment) { 'development' }
+          BackgroundQueue::Utils.stub(:current_environment) { 'development' }
         end
         it "calls load using map" do
           BackgroundQueue::Config.should_receive(:load_hash).with({'a'=>'b'}, :path_that_exists) { true }

data/spec/background_queue/server_lib/integration/error_handling_spec.rb

@@ -0,0 +1,85 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../../spec_helper')
+require 'background_queue_server'
+require 'background_queue'
+require 'background_queue_worker'
+require 'net/http'
+
+require 'rubygems'
+gem "rufus-scheduler"
+
+#unless defined? Rails
+  module Rails; 
+    def self.env
+      "development"
+    end
+  end
+#end
+
+describe "Error Handling Test" do
+
+  context "same task" do
+  
+    it "will add tasks to an error list" do
+
+      config_path = File.expand_path(File.dirname(__FILE__) + '/../../../resources/config.yml')
+      
+      server = BackgroundQueue::ServerLib::Server.new
+
+      thread = Thread.new {
+        server.start(:config=>config_path, :skip_pid=>true, :command=>:run, :log_file=>"/tmp/bq.log", :log_level=>'debug')
+      }
+
+      while server.event_server.nil? || !server.event_server.running
+        sleep(0.1)
+      end
+      ss = nil
+      begin
+
+        meth_name = nil
+        ss = TestWorkerServer.new(8001)
+        attempt_count = 0
+        ss.start(Proc.new { |controller|
+          meth_name = controller.request.request_method
+          attempt_count += 1
+          if attempt_count < 3
+            controller.render :text=>"ARGH", :type=>"text/text", :status=>500
+          else
+            controller.render :text=>{:percent=>100, :caption=>"Done"}.to_json, :type=>"text/text"  
+          end
+        })
+        
+        
+        client_config_path = File.expand_path(File.dirname(__FILE__) + '/../../../resources/config-client.yml')
+        
+        client = BackgroundQueue::Client.new(client_config_path)
+        
+  
+        job_handle = client.add_task(:some_worker, :owner_id, :job_id, :task_id, 2, {:something=>:else}, {:domain=>"www.example.com"} )
+        
+        
+        ss.wait_to_be_called.should be_true
+        
+        #add same task while in existing in error state
+        job_handle = client.add_task(:some_worker, :owner_id, :job_id, :task_id, 2, {:something=>:else}, {:domain=>"www.example.com"} )
+        
+        ss.wait_to_be_called.should be_true
+        ss.wait_to_be_called.should be_true
+        
+        result = client.get_status(job_handle)
+  
+        result.code.should eq(:status)
+        result.args[:percent].should eq(100)
+        result.args[:caption].should eq('Done')
+  
+        
+      ensure
+        ss.stop unless ss.nil?
+        server.stop
+        
+        thread.join
+      end
+    end
+    
+  end
+  
+end

data/spec/background_queue/server_lib/integration/full_test_spec.rb

@@ -92,7 +92,7 @@ describe "Full Test" do
       while server.event_server.nil? || !server.event_server.running
         sleep(0.1)
       end
-      
+      ss = nil
       begin
 
         task_pos = 0
@@ -149,7 +149,7 @@ describe "Full Test" do
         meth_name.should eq("POST")
         
         stats = client.get_stats(job_handle.server)
-        stats[:running].should eq(1)   
+        stats[:running].should eq(2)   
         stats[:run_tasks].should eq(1)
         
         #pp server
@@ -180,8 +180,9 @@ describe "Full Test" do
         stats[:running].should eq(0)   
         stats[:run_tasks].should eq(3)
         
-        ss.stop
+        
       ensure
+        ss.stop unless ss.nil?
         server.stop
         thread.join
       end
@@ -243,5 +244,77 @@ describe "Full Test" do
       end
     end
     
+    
+    it "will keep track of summary" do
+
+      config_path = File.expand_path(File.dirname(__FILE__) + '/../../../resources/config.yml')
+      BackgroundQueue::Worker::Config.worker_path = File.expand_path(File.dirname(__FILE__) + '/../../../resources/')
+      BackgroundQueue::Worker::Config.secret = "this_is_used_to_make_sure_it_is_secure"
+      
+      server = BackgroundQueue::ServerLib::Server.new
+
+      thread = Thread.new {
+        server.start(:config=>config_path, :skip_pid=>true, :command=>:run, :log_file=>"/tmp/bq.log", :log_level=>'debug')
+      }
+
+      while server.event_server.nil? || !server.event_server.running
+        sleep(0.1)
+      end
+      
+      begin
+
+        count = 0
+        summary_data = nil 
+        meth_name = nil
+        ss = TestWorkerServer.new(8001, true)
+        ss.start(Proc.new { |controller|
+            controller.class.send(:include, BackgroundQueue::Worker::Calling)
+            controller.run_worker
+        })
+        
+        
+        client_config_path = File.expand_path(File.dirname(__FILE__) + '/../../../resources/config-client.yml')
+        
+        client = BackgroundQueue::Client.new(client_config_path)
+        
+  
+        job_handle = client.add_tasks(:summary_worker, :owner_id, :job_id, [[:task1_id, {:test_id=>1}], [:task2_id, {:test_id=>2}], [:task3_id, {:mode=>"summary"}, {:send_summary=>true, :weight=>10, :exclude=>true, :synchronous=>true}]], 2, {:something=>:else}, {:domain=>"www.example.com"} )
+  
+        ss.allow_to_be_called
+        ss.wait_to_be_called.should be_true
+        
+        result = client.get_status(job_handle)
+  
+        result.code.should eq(:status)
+        result.args[:percent].should eq(45)
+        result.args[:caption].should eq('Done (2/2)')
+        result.args[:meta].should eq({'test_meta'=>"something"})
+        
+        ss.allow_to_be_called
+        ss.wait_to_be_called.should be_true
+        
+        result = client.get_status(job_handle)
+  
+        result.code.should eq(:status)
+        result.args[:percent].should eq(90)
+        result.args[:caption].should eq('Done (2/2)')
+        
+        ss.allow_to_be_called
+        ss.wait_to_be_called.should be_true
+        
+        result = client.get_status(job_handle)
+  
+        result.code.should eq(:status)
+        result.args[:percent].should eq(100)
+        result.args[:caption].should eq('Done')
+
+        ss.stop
+      ensure
+        server.stop
+        
+        thread.join
+      end
+    end
+    
   end
 end