background_queue 0.2.0 → 0.3.0

data/Gemfile

@@ -6,6 +6,8 @@ gem "json"
 gem "rufus-scheduler"
 gem "eventmachine", "~>0.12.10"
 gem "ipaddress", "~> 0.8.0"
+gem "ipaddress", "~> 0.8.0"
+gem "algorithms", "~> 0.5.0"
   
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.3.0

data/background_queue.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "background_queue"
-  s.version = "0.2.0"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["MarkPent"]
-  s.date = "2012-10-09"
+  s.date = "2012-10-17"
   s.description = "Organise background tasks so they will not overload the machine(s) running the tasks, while still giving a fair, balanced allocation of running time to members in the queue"
   s.email = "mark.pent@gmail.com"
   s.executables = ["bg_queue"]
@@ -40,6 +40,7 @@ Gem::Specification.new do |s|
     "lib/background_queue/config.rb",
     "lib/background_queue/server_lib/balanced_queue.rb",
     "lib/background_queue/server_lib/config.rb",
+    "lib/background_queue/server_lib/error_task_list.rb",
     "lib/background_queue/server_lib/event_connection.rb",
     "lib/background_queue/server_lib/event_server.rb",
     "lib/background_queue/server_lib/job.rb",
@@ -73,6 +74,7 @@ Gem::Specification.new do |s|
     "spec/background_queue/config_spec.rb",
     "spec/background_queue/server_lib/balanced_queue_spec.rb",
     "spec/background_queue/server_lib/config_spec.rb",
+    "spec/background_queue/server_lib/error_task_list_spec.rb",
     "spec/background_queue/server_lib/event_connection_spec.rb",
     "spec/background_queue/server_lib/event_server_spec.rb",
     "spec/background_queue/server_lib/integration/full_test_spec.rb",
@@ -95,7 +97,6 @@ Gem::Specification.new do |s|
     "spec/background_queue/worker/calling_spec.rb",
     "spec/background_queue/worker/environment_spec.rb",
     "spec/background_queue/worker/worker_loader_spec.rb",
-    "spec/background_queue_spec.rb",
     "spec/resources/config-client.yml",
     "spec/resources/config-serialize.yml",
     "spec/resources/config.yml",
@@ -124,6 +125,8 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<rufus-scheduler>, [">= 0"])
       s.add_runtime_dependency(%q<eventmachine>, ["~> 0.12.10"])
       s.add_runtime_dependency(%q<ipaddress>, ["~> 0.8.0"])
+      s.add_runtime_dependency(%q<ipaddress>, ["~> 0.8.0"])
+      s.add_runtime_dependency(%q<algorithms>, ["~> 0.5.0"])
       s.add_development_dependency(%q<rspec>, [">= 2.9.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.8.3"])
       s.add_development_dependency(%q<yard>, ["~> 0.7"])
@@ -135,6 +138,8 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<rufus-scheduler>, [">= 0"])
       s.add_dependency(%q<eventmachine>, ["~> 0.12.10"])
       s.add_dependency(%q<ipaddress>, ["~> 0.8.0"])
+      s.add_dependency(%q<ipaddress>, ["~> 0.8.0"])
+      s.add_dependency(%q<algorithms>, ["~> 0.5.0"])
       s.add_dependency(%q<rspec>, [">= 2.9.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.8.3"])
       s.add_dependency(%q<yard>, ["~> 0.7"])
@@ -147,6 +152,8 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<rufus-scheduler>, [">= 0"])
     s.add_dependency(%q<eventmachine>, ["~> 0.12.10"])
     s.add_dependency(%q<ipaddress>, ["~> 0.8.0"])
+    s.add_dependency(%q<ipaddress>, ["~> 0.8.0"])
+    s.add_dependency(%q<algorithms>, ["~> 0.5.0"])
     s.add_dependency(%q<rspec>, [">= 2.9.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.8.3"])
     s.add_dependency(%q<yard>, ["~> 0.7"])

data/lib/background_queue/server_lib/error_task_list.rb

@@ -0,0 +1,116 @@
+require 'algorithms'
+module BackgroundQueue::ServerLib
+  
+  class ErrorTaskList
+  
+    attr_reader :tasks
+    attr_reader :task_count
+    
+    def initialize(server)
+      @server = server
+      @tasks = Containers::RBTreeMap.new
+      @mutex = Mutex.new
+      @current_next_at = nil
+      @current_runner = nil
+      @task_count = 0
+    end
+    
+    def add_task(task)
+      task.increment_error_count
+      delay = calculate_delay(task.get_error_count)             
+      add_item(task, Time.now.to_i + delay)
+    end
+    
+    def calculate_delay(error_count)
+      delay = error_count * error_count
+      delay > 120 ? 120 : delay
+    end
+    
+    def add_item(task, time_at)
+      @mutex.synchronize {
+        existing = @tasks[time_at]
+        if existing.nil?
+          existing = []
+          @tasks[time_at] = existing
+        end
+        existing << task
+        @task_count += 1
+        queue_next_event(time_at)
+      }
+      @server.logger.debug("Task #{task.id} queued to retry in #{time_at - Time.now.to_f} seconds")
+    end
+    
+    def queue_next_event(time_at)
+      if @current_next_at.nil? || @current_next_at > time_at
+        @current_runner.cancel if @current_runner
+        @current_next_at = time_at
+        @current_runner = BackgroundQueue::ServerLib::ErrorTaskList::RunAt.new(time_at) {
+          self.next_event
+        }
+      end
+    end
+    
+    def next_event
+      @mutex.synchronize {
+        @current_runner = nil
+        @current_next_at = nil
+        while @tasks.size > 0 && @tasks.min_key < (Time.now.to_f + 0.1)
+          next_tasks = @tasks.delete_min
+          for task in next_tasks
+            @server.task_queue.finish_task(task)
+            @server.task_queue.add_task(task)
+            @task_count -= 1
+          end
+        end
+        queue_next_event(@tasks.min_key) if @tasks.size > 0
+      }
+    end
+    
+    def flush
+      @server.logger.debug("Flushing #{@tasks.size} tasks from error list")
+      @current_runner.cancel if @current_runner
+      @current_runner = nil
+      @mutex.synchronize {
+        while @tasks.size > 0
+          next_tasks = @tasks.delete_min
+          for task in next_tasks
+            @server.task_queue.finish_task(task)
+            @server.task_queue.add_task(task)
+            @task_count -= 1
+          end
+        end
+      }
+    end
+    
+    def wait_for_event
+      runner = @current_runner
+      runner.wait_for_run if runner
+    end
+    
+    class RunAt
+      #i dont care about thread safety: it doesnt matter if an event runs twice.
+      def initialize(at, &block)
+        @running = true
+        @th = Thread.new {
+          delay = at - Time.now.to_f 
+          sleep(delay)
+          if @running
+            block.call
+            @running = false
+          end
+        }
+      end
+      
+      def cancel
+        return false if !@running
+        @running = false
+        @th.run #wake the sleep up
+      end
+      
+      def wait_for_run
+        @th.join
+      end
+      
+    end
+  end
+end

data/lib/background_queue/server_lib/event_connection.rb

@@ -49,6 +49,7 @@ module BackgroundQueue::ServerLib
         send_result(result)
       rescue Exception=>e
         @server.logger.error("Error processing command: #{e.message}")
+        @server.logger.debug(e.backtrace.join("\n"))
         send_error(e.message)
       end
     end

data/lib/background_queue/server_lib/job.rb

@@ -1,6 +1,7 @@
 module BackgroundQueue::ServerLib
   class Job < PriorityQueue
   
+    
     attr_accessor :id
     attr_reader :running_status
     attr_reader :running_ordered_status
@@ -79,6 +80,10 @@ module BackgroundQueue::ServerLib
       pop
     end
     
+    def remove_item(item)
+      remove(item)
+    end
+    
     def finish_item(item)
       @synchronous_count-=1 if item.synchronous?
     end

data/lib/background_queue/server_lib/server.rb

@@ -7,6 +7,7 @@ module BackgroundQueue::ServerLib
     attr_accessor :config
     attr_accessor :thread_manager
     attr_accessor :task_queue
+    attr_accessor :error_tasks
     attr_accessor :event_server
     attr_accessor :workers
     attr_accessor :jobs
@@ -37,7 +38,7 @@ module BackgroundQueue::ServerLib
       OptionParser.new do |opts|
         opts.banner = "Usage: server command [options]"
         case options[:command]
-          when :start, :test
+          when :start, :test, :run
             opts.on("-c", "--config PATH", "Configuration Path") do |cp|
               options[:config] = cp
             end
@@ -60,6 +61,10 @@ module BackgroundQueue::ServerLib
         opts.on("-e", "--environment [RAILS_ENV]", "testing/development/production (development)") do |env|
           env_to_load = env
         end
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts opts
+          exit
+        end
       end.parse!(argv)
       
       ENV['RAILS_ENV']=env_to_load
@@ -254,6 +259,8 @@ module BackgroundQueue::ServerLib
       
       @event_server = BackgroundQueue::ServerLib::EventServer.new(self)
       
+      @error_tasks = BackgroundQueue::ServerLib::ErrorTaskList.new(self)
+      
       @jobs = BackgroundQueue::ServerLib::JobRegistry.new
       
       load_tasks(config.task_file)
@@ -265,6 +272,7 @@ module BackgroundQueue::ServerLib
       @running = false
       @event_server.stop
       @thread_manager.wait(timeout_secs)
+      @error_tasks.flush
       save_tasks(config.task_file)
     end
     

data/lib/background_queue/server_lib/task.rb

@@ -68,6 +68,20 @@ module BackgroundQueue::ServerLib
       @options[:initial_progress_caption]
     end
     
+    def get_error_count
+      if @error_count.nil?
+        0
+      else
+        @error_count
+      end
+    end
+    
+    
+    def increment_error_count
+      @error_count = get_error_count + 1
+    end
+    
+    
     def set_worker_status(status)
       raise "Task without job set" if @job.nil?
       status[:task_id] = self.id

data/lib/background_queue/server_lib/thread_manager.rb

@@ -111,6 +111,16 @@ class BackgroundQueue::ServerLib::ThreadManager
           #ignore
         end
       end
+      
+      begin
+        Timeout::timeout(timeout_limit) {
+          for thread in @threads
+            thread.join
+          end
+        }
+      rescue Timeout::Error => te2
+        @server.logger.error("Timeout while waiting for forced stop threads to finish")
+      end
     end
   end
   

data/lib/background_queue/server_lib/worker_client.rb

@@ -19,11 +19,17 @@ module BackgroundQueue::ServerLib
             read_response(worker, response, task)
           end
         end
-        true
+        return :ok
+      rescue BackgroundQueue::ServerLib::WorkerError => we
+        @server.logger.error("Worker Error sending request #{task.id} to worker: #{e.message}")
+        return :worker_error
+      rescue BackgroundQueue::ServerLib::ThreadManager::ForcedStop => fe
+        @server.logger.error("Thread stop while sending request #{task.id} to worker: #{fe.message}")
+        return :stop
       rescue Exception=>e
         @server.logger.error("Error sending request #{task.id} to worker: #{e.message}")
         @server.logger.debug(e.backtrace.join("\n"))
-        return false
+        return :fatal_error
       end
     end
     
@@ -49,7 +55,7 @@ module BackgroundQueue::ServerLib
           @prev_chunk = nil
         end
       else
-        raise "Invalid response code (#{http_response.code}) when calling #{worker.uri.to_s}"
+        raise BackgroundQueue::ServerLib::WorkerError, "Invalid response code (#{http_response.code}) when calling #{worker.uri.to_s}"
       end
     end
     
@@ -91,4 +97,8 @@ module BackgroundQueue::ServerLib
     end
     
   end
+  
+  class WorkerError < Exception
+    
+  end
 end

data/lib/background_queue/server_lib/worker_thread.rb

@@ -31,24 +31,26 @@ module BackgroundQueue::ServerLib
           Kernel.sleep(1) unless !@server.running?
         else
           client = build_client
-          if client.send_request(worker, task, @server.config.secret)
+          result = client.send_request(worker, task, @server.config.secret)
+          if result == :ok
             @server.logger.debug("called worker for task #{task.id}")
             @server.workers.finish_using_worker(worker, true)
             @server.task_queue.finish_task(task)
             @server.change_stat(:running, -1)
             @server.change_stat(:run_tasks, 1)
-            
             return true
           else
             @server.logger.debug("failed calling worker for task #{task.id}")
-            @server.workers.finish_using_worker(worker, false)
-            error_count += 1
-            if error_count > 5
-              @server.logger.debug("error count exceeded for task #{task.id}, returning to queue")
-              @server.task_queue.finish_task(task)
-              @server.task_queue.add_task(task)
-              return true
-            end
+            @server.workers.finish_using_worker(worker, result == :worker_error)
+            @server.error_tasks.add_task(task)
+            return result != :stop
+            #error_count += 1
+            #if error_count > 5
+            #  @server.logger.debug("error count exceeded for task #{task.id}, returning to queue")
+            #  @server.task_queue.finish_task(task)
+            #  @server.task_queue.add_task(task)
+            #  return true
+            #end
           end
         end
       end

data/lib/background_queue/worker/config.rb

@@ -1,6 +1,8 @@
 module BackgroundQueue::Worker
   class Config
       
+    @@worker_path = nil
+    
     def self.secret=(auth)
       @@secret = auth
     end

data/lib/background_queue_server.rb

@@ -1,5 +1,6 @@
 require "background_queue/utils"
 require "background_queue/config"
+require "background_queue/command"
 require "background_queue/server_lib/lru"
 require "background_queue/server_lib/config"
 require "background_queue/server_lib/priority_queue"
@@ -19,3 +20,4 @@ require "background_queue/server_lib/worker_thread"
 require "background_queue/server_lib/server"
 require "background_queue/server_lib/event_server"
 require "background_queue/server_lib/event_connection"
+require "background_queue/server_lib/error_task_list"

data/spec/background_queue/server_lib/error_task_list_spec.rb

@@ -0,0 +1,142 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue_server'
+
+
+describe BackgroundQueue::ServerLib::ErrorTaskList do
+
+  let(:server) {
+    SimpleServer.new
+  }
+  
+  subject { 
+    BackgroundQueue::ServerLib::ErrorTaskList.new(server)
+  }
+  
+  let(:task) {
+    double("task", :get_error_count=>1, :id=>:task1)
+  }
+  let(:task2) {
+    double("task2", :get_error_count=>1, :id=>:task2)
+  }
+  let(:task3) {
+    double("task3", :get_error_count=>1, :id=>:task3)
+  }
+  let(:task4) {
+    double("task4", :get_error_count=>1, :id=>:task4)
+  }
+  
+  
+  context "#add_task" do
+    it "will calculate the delay then add it to the list using it" do
+      subject.should_receive(:calculate_delay).with(1).and_return(1)
+      subject.should_receive(:add_item).with(task, 101)
+      Time.any_instance.should_receive(:to_i).and_return(100)
+      task.should_receive(:increment_error_count)
+      subject.add_task(task)
+    end
+  end
+  
+  context "#calculate_delay" do
+    it "will square the error count" do
+      subject.calculate_delay(1).should eq(1)
+      subject.calculate_delay(2).should eq(4)
+      subject.calculate_delay(3).should eq(9)
+      subject.calculate_delay(10).should eq(100)
+    end
+    
+    it "will use a threshold (120)" do
+      subject.calculate_delay(50).should eq(120)
+    end
+  end
+  
+  context "#add_item" do
+    it "will add an item using the scheduled time as key" do
+      subject.should_receive(:queue_next_event).with(10)
+      subject.add_item(task, 10)
+      subject.tasks.size.should eq(1)
+      subject.tasks.min_key.should be(10)
+      subject.tasks.delete_min.should eq([task])
+    end
+    
+    it "will add the task to an event at the same time" do
+      subject.should_receive(:queue_next_event).twice.with(10)
+      subject.should_receive(:queue_next_event).with(20)
+      subject.add_item(task, 10)
+      subject.add_item(task2, 10)
+      subject.add_item(task3, 20)
+      subject.tasks.size.should eq(2)
+      subject.tasks.min_key.should be(10)
+      subject.tasks.delete_min.should eq([task, task2])
+      subject.tasks.min_key.should be(20)
+      subject.tasks.delete_min.should eq([task3])
+    end
+  end
+  
+  context "#queue_next_event" do
+    it "will queue a new time if its closer then the last" do
+      timer = double("timer")
+      timer.should_receive(:cancel)
+      BackgroundQueue::ServerLib::ErrorTaskList::RunAt.should_receive(:new).with(10).and_return(timer)
+      BackgroundQueue::ServerLib::ErrorTaskList::RunAt.should_receive(:new).with(5).and_return(timer)
+      subject.queue_next_event(10)
+      subject.queue_next_event(5)
+    end
+    
+    it "will not queue a new event if its furhter away" do
+      timer = double("timer")
+      BackgroundQueue::ServerLib::ErrorTaskList::RunAt.should_receive(:new).with(10).and_return(timer)
+      subject.queue_next_event(10)
+      subject.queue_next_event(20)
+    end
+    
+    it "will fire an event" do
+      subject.should_receive(:next_event)
+      subject.queue_next_event(Time.now.to_f + 0.1)
+      subject.wait_for_event
+    end
+  end
+  
+  context "#next_event" do
+    it "will get all the tasks before the current time and re-add them" do
+      subject.tasks[Time.now.to_f] = [task, task2]
+      subject.tasks[Time.now.to_f + 0.01] = [task3]
+      task_at = Time.now.to_f + 1.0
+      subject.tasks[task_at] = [task4]
+      server.task_queue = double("tq")
+      server.task_queue.should_receive(:finish_task).with(task)
+      server.task_queue.should_receive(:add_task).with(task)
+      server.task_queue.should_receive(:finish_task).with(task2)
+      server.task_queue.should_receive(:add_task).with(task2)
+      server.task_queue.should_receive(:finish_task).with(task3)
+      server.task_queue.should_receive(:add_task).with(task3)
+      subject.should_receive(:queue_next_event).with(task_at)
+      subject.next_event
+      subject.tasks.size.should eq(1)
+    end
+  end
+  
+  context "RunAt" do
+    it "will run a task at the specificied time" do
+      run_at = Time.now.to_f + 0.1
+      when_run = 0
+      runner = BackgroundQueue::ServerLib::ErrorTaskList::RunAt.new(run_at) {
+        when_run = Time.now.to_f
+      }
+      runner.wait_for_run
+      when_run.should_not eq(0)
+      when_run.to_i.should eq(run_at.to_i)
+    end
+    
+    it "can be cancelled" do
+      run_at = Time.now.to_f + 0.5
+      has_run = false
+      runner = BackgroundQueue::ServerLib::ErrorTaskList::RunAt.new(run_at) {
+        has_run = true
+      }
+      sleep(0.05)
+      runner.cancel.should be_true
+      runner.wait_for_run
+      has_run.should be_false
+    end
+  end
+end

data/spec/background_queue/server_lib/integration/queue_integration_spec.rb

@@ -72,7 +72,7 @@ describe "Queue Integration" do
       
       task = subject.next_item
       task.id.should be(:task_id)
-      subject.next_item.should be_nil
+      subject.next_item.should be_nil 
       subject.finish_item(task)
       subject.next_item.id.should eq(:task_id2)
       
@@ -94,5 +94,29 @@ describe "Queue Integration" do
       
     end
   end
+  
+  
+  
+  context "Duplicate Tasks" do
+    it "will replace a duplicate task not running" do
+      subject.add_task(SimpleTask.new(:owner_id, :job_id, :task_id, 3, {:synchronous=>true}))
+      subject.add_task(SimpleTask.new(:owner_id, :job_id, :task_id, 3, {:synchronous=>true}))
+      task = subject.next_item
+      task.id.should be(:task_id)
+      subject.next_item.should be_nil 
+    end
+    
+    it "will queue a duplicate task that is running" do
+      subject.add_task(SimpleTask.new(:owner_id, :job_id, :task_id, 3, {:synchronous=>true}))
+      task = subject.next_item
+      task.id.should be(:task_id)
+      subject.next_item.should be_nil 
+      subject.add_task(SimpleTask.new(:owner_id, :job_id, :task_id, 3, {:synchronous=>true}))
+      subject.next_item.should be_nil 
+      subject.finish_item(task)
+      task = subject.next_item
+      task.id.should be(:task_id)
+    end
+  end
 
 end

data/spec/background_queue/server_lib/worker_client_spec.rb

@@ -40,7 +40,7 @@ describe BackgroundQueue::ServerLib::WorkerClient do
       task = SimpleTask.new(:owner_id, :job_id, :task_id, 3, { :domain=>"www.example.com" })
       subject.should_receive(:build_request).with(uri, task, "auth").and_return(:post_request)
       Net::HTTP.should_receive(:start).with("127.0.0.1", 3000).and_raise("connection error")
-      subject.send_request(worker_config, task, "auth").should be_false
+      subject.send_request(worker_config, task, "auth").should eq(:fatal_error)
     end
   end
   
@@ -112,7 +112,7 @@ describe BackgroundQueue::ServerLib::WorkerClient do
   
   context "can handle thread cancelling" do
     #this can cause issues with other tests....
-    xit "will return false if exception raised" do
+    it "will return false if server stopped", :can_be_flakey=>true do
       
       mutex = Mutex.new
       resource = ConditionVariable.new
@@ -122,66 +122,69 @@ describe BackgroundQueue::ServerLib::WorkerClient do
 
       run_request = false
       ss = TestWorkerServer.new(8001)
-      ss.start(Proc.new { |controller|
-        #puts "in proc"
-          
-        mutex2.synchronize {
-          resource2.signal
+      begin
+        ss.start(Proc.new { |controller|
+          #puts "in proc"
+            
+          mutex2.synchronize {
+            resource2.signal
+          }
+          #puts "in proc: waiting" 
+          mutex.synchronize {
+            resource.wait(mutex)
+          }
+          #puts "waited"
+          run_request = true
+          controller.render :text =>{:percent=>100, :caption=>"Done"}.to_json, :type=>"text/text"
+        })
+        
+        
+        uri = URI("http://127.0.0.1:8001/background_queue")
+        worker_config = BackgroundQueue::ServerLib::Worker.new(uri)
+        task = SimpleTask.new(:owner_id, :job_id, :task_id, 3, { :domain=>"www.example.com" })
+        
+        call_result = nil
+        t1 = Thread.new {
+          #puts "calling"
+          begin
+            status = Timeout::timeout(2) {
+              call_result = subject.send_request(worker_config, task, "abcd")
+             # puts "called"
+            }
+          rescue Timeout::Error=>te
+            #puts "timeout"
+            call_result = :timeout
+          end
+          mutex2.synchronize {
+            resource2.signal
+          }
         }
-        #puts "in proc: waiting" 
-        mutex.synchronize {
-          resource.wait(mutex)
+  
+        #wait until we know the request has been sent and is processing
+        mutex2.synchronize {
+          resource2.wait(mutex2)
         }
-        #puts "waited"
-        run_request = true
-        controller.render :text =>{:percent=>100, :caption=>"Done"}.to_json, :type=>"text/text"
-      })
-      
-      
-      uri = URI("http://127.0.0.1:8001/background_queue")
-      worker_config = BackgroundQueue::ServerLib::Worker.new(uri)
-      task = SimpleTask.new(:owner_id, :job_id, :task_id, 3, { :domain=>"www.example.com" })
-      
-      call_result = nil
-      t1 = Thread.new {
-        #puts "calling"
-        begin
-          status = Timeout::timeout(2) {
-            call_result = subject.send_request(worker_config, task, "abcd")
-           # puts "called"
-          }
-        rescue Timeout::Error=>te
-          #puts "timeout"
-          call_result = :timeout
-        end
+        run_request.should be_false
+        #puts "cancelling"
+        t1.raise BackgroundQueue::ServerLib::ThreadManager::ForcedStop.new("Timeout when forcing threads to stop")
+        
+        #puts "canceled"
+        #wait until we know the request has been cancelled
         mutex2.synchronize {
-          resource2.signal
+          resource2.wait(mutex2)
         }
-      }
-
-      #wait until we know the request has been sent and is processing
-      mutex2.synchronize {
-        resource2.wait(mutex2)
-      }
-      run_request.should be_false
-      #puts "cancelling"
-      t1.raise "Cancelling From Thread"
-      
-      #puts "canceled"
-      #wait until we know the request has been cancelled
-      mutex2.synchronize {
-        resource2.wait(mutex2)
-      }
-      call_result.should be_false
-      run_request.should be_false
-
-      mutex.synchronize {
-        resource.signal
-      }
-     
-      
-      t1.join
-      ss.stop
+        call_result.should eq(:stop)
+        run_request.should be_false
+  
+        mutex.synchronize {
+          resource.signal
+        }
+       
+        
+        t1.join
+      ensure
+        ss.stop
+      end
       
     end
     

data/spec/background_queue/server_lib/worker_thread_spec.rb

@@ -11,9 +11,12 @@ describe BackgroundQueue::ServerLib::WorkerThread do
   let(:workers) {
     double("workers")
   }
+  let(:error_tasks) {
+    double("error_tasks")
+  }
   
   let(:server) {
-    SimpleServer.new(:task_queue=>balanced_queue, :workers=>workers, :config=>double("conf", :secret=>:secret))
+    SimpleServer.new(:task_queue=>balanced_queue, :workers=>workers, :config=>double("conf", :secret=>:secret), :error_tasks=>error_tasks)
   }
   
   subject { BackgroundQueue::ServerLib::WorkerThread.new(server) }
@@ -48,31 +51,38 @@ describe BackgroundQueue::ServerLib::WorkerThread do
       server.stub('running?'=>true)
       worker = double("worker")
       task = DefaultTask.new
-      BackgroundQueue::ServerLib::WorkerClient.any_instance.should_receive(:send_request).with(worker, task, :secret).and_return(true)
+      worker_client = double("worker_client")
+      subject.should_receive(:build_client).and_return(worker_client)
+      worker_client.should_receive(:send_request).with(worker, task, :secret).and_return(:ok)
       server.workers.should_receive(:get_next_worker).and_return(worker)
       server.workers.should_receive(:finish_using_worker).with(worker, true)
       server.task_queue.should_receive(:finish_task).with(task)
       subject.call_worker(task).should be_true
     end
     
-    it "will keep trying if the worker fails" do
+    it "will regegister as a failed worker if send_request returns :fatal_error" do
       server.stub('running?'=>true)
       worker = double("worker")
       task = DefaultTask.new
-      count = 0
-      worker_client1 = double("w1")
-      worker_client1.should_receive(:send_request).with(worker, task, :secret).and_return(false)
-      worker_client2 = double("w2")
-      worker_client2.should_receive(:send_request).with(worker, task, :secret).and_return(true)
-      
-      subject.should_receive(:build_client).twice {
-        count += 1
-        count == 1 ? worker_client1 : worker_client2
-      }
-      server.workers.should_receive(:get_next_worker).twice.and_return(worker)
+      server.error_tasks.should_receive(:add_task).with(task)
+      worker_client = double("worker_client")
+      subject.should_receive(:build_client).and_return(worker_client)
+      worker_client.should_receive(:send_request).with(worker, task, :secret).and_return(:fatal_error)
+      server.workers.should_receive(:get_next_worker).and_return(worker)
       server.workers.should_receive(:finish_using_worker).with(worker, false)
+      subject.call_worker(task).should be_true
+    end
+    
+    it "will regegister as a ok worker if send_request returns :worker_error" do
+      server.stub('running?'=>true)
+      worker = double("worker")
+      task = DefaultTask.new
+      server.error_tasks.should_receive(:add_task).with(task)
+      worker_client = double("worker_client")
+      subject.should_receive(:build_client).and_return(worker_client)
+      worker_client.should_receive(:send_request).with(worker, task, :secret).and_return(:worker_error)
+      server.workers.should_receive(:get_next_worker).and_return(worker)
       server.workers.should_receive(:finish_using_worker).with(worker, true)
-      server.task_queue.should_receive(:finish_task).with(task)
       subject.call_worker(task).should be_true
     end
     
@@ -86,7 +96,7 @@ describe BackgroundQueue::ServerLib::WorkerThread do
         count == 1 ? nil : worker
       }
       Kernel.should_receive(:sleep).with(1)
-      BackgroundQueue::ServerLib::WorkerClient.any_instance.should_receive(:send_request).with(worker, task, :secret).and_return(true)
+      BackgroundQueue::ServerLib::WorkerClient.any_instance.should_receive(:send_request).with(worker, task, :secret).and_return(:ok)
       server.workers.should_receive(:finish_using_worker).with(worker, true)
       server.task_queue.should_receive(:finish_task).with(task)
       subject.call_worker(task).should be_true

data/spec/spec_helper.rb

@@ -9,7 +9,7 @@ Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
 Dir["#{File.dirname(__FILE__)}/shared/**/*.rb"].each {|f| require f}
 
 RSpec.configure do |config|
-  
+  #config.filter_run_excluding :can_be_flakey => true
 end
 
 

data/spec/support/simple_server.rb

@@ -6,6 +6,7 @@ class SimpleServer
   attr_accessor :workers
   attr_accessor :jobs
   attr_accessor :logger
+  attr_accessor :error_tasks
   
   def initialize(options={})
     @logger = Logger.new("/dev/null")
@@ -15,6 +16,7 @@ class SimpleServer
     @event_server = options[:event_server]
     @workers = options[:workers]
     @jobs = options[:jobs]
+    @error_tasks = options[:error_tasks]
   end
   
   def running?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: background_queue
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-09 00:00:00.000000000 Z
+date: 2012-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &21144380 !ruby/object:Gem::Requirement
+  requirement: &19238880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *21144380
+  version_requirements: *19238880
 - !ruby/object:Gem::Dependency
   name: rufus-scheduler
-  requirement: &21156100 !ruby/object:Gem::Requirement
+  requirement: &20184220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *21156100
+  version_requirements: *20184220
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &21161240 !ruby/object:Gem::Requirement
+  requirement: &20182860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.12.10
   type: :runtime
   prerelease: false
-  version_requirements: *21161240
+  version_requirements: *20182860
 - !ruby/object:Gem::Dependency
   name: ipaddress
-  requirement: &21156960 !ruby/object:Gem::Requirement
+  requirement: &20180600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,32 @@ dependencies:
         version: 0.8.0
   type: :runtime
   prerelease: false
-  version_requirements: *21156960
+  version_requirements: *20180600
+- !ruby/object:Gem::Dependency
+  name: ipaddress
+  requirement: &20178440 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *20178440
+- !ruby/object:Gem::Dependency
+  name: algorithms
+  requirement: &20177140 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *20177140
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &21170000 !ruby/object:Gem::Requirement
+  requirement: &19623560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +87,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *21170000
+  version_requirements: *19623560
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &21166880 !ruby/object:Gem::Requirement
+  requirement: &19621540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +98,10 @@ dependencies:
         version: 1.8.3
   type: :development
   prerelease: false
-  version_requirements: *21166880
+  version_requirements: *19621540
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &21166160 !ruby/object:Gem::Requirement
+  requirement: &19617840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,10 +109,10 @@ dependencies:
         version: '0.7'
   type: :development
   prerelease: false
-  version_requirements: *21166160
+  version_requirements: *19617840
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &21164920 !ruby/object:Gem::Requirement
+  requirement: &19615920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -98,10 +120,10 @@ dependencies:
         version: '3.12'
   type: :development
   prerelease: false
-  version_requirements: *21164920
+  version_requirements: *19615920
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &21180500 !ruby/object:Gem::Requirement
+  requirement: &20601320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -109,10 +131,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *21180500
+  version_requirements: *20601320
 - !ruby/object:Gem::Dependency
   name: redcarpet
-  requirement: &21179640 !ruby/object:Gem::Requirement
+  requirement: &20599700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -120,7 +142,7 @@ dependencies:
         version: 2.1.1
   type: :development
   prerelease: false
-  version_requirements: *21179640
+  version_requirements: *20599700
 description: Organise background tasks so they will not overload the machine(s) running
   the tasks, while still giving a fair, balanced allocation of running time to members
   in the queue
@@ -154,6 +176,7 @@ files:
 - lib/background_queue/config.rb
 - lib/background_queue/server_lib/balanced_queue.rb
 - lib/background_queue/server_lib/config.rb
+- lib/background_queue/server_lib/error_task_list.rb
 - lib/background_queue/server_lib/event_connection.rb
 - lib/background_queue/server_lib/event_server.rb
 - lib/background_queue/server_lib/job.rb
@@ -187,6 +210,7 @@ files:
 - spec/background_queue/config_spec.rb
 - spec/background_queue/server_lib/balanced_queue_spec.rb
 - spec/background_queue/server_lib/config_spec.rb
+- spec/background_queue/server_lib/error_task_list_spec.rb
 - spec/background_queue/server_lib/event_connection_spec.rb
 - spec/background_queue/server_lib/event_server_spec.rb
 - spec/background_queue/server_lib/integration/full_test_spec.rb
@@ -209,7 +233,6 @@ files:
 - spec/background_queue/worker/calling_spec.rb
 - spec/background_queue/worker/environment_spec.rb
 - spec/background_queue/worker/worker_loader_spec.rb
-- spec/background_queue_spec.rb
 - spec/resources/config-client.yml
 - spec/resources/config-serialize.yml
 - spec/resources/config.yml
@@ -238,7 +261,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 2926344861987835839
+      hash: 613144620285186166
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:

data/spec/background_queue_spec.rb

@@ -1,7 +0,0 @@
-require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
-
-describe "BackgroundQueue" do
-  xit "does nothing yet" do
-
-  end
-end