background_queue 0.3.0 → 0.6.0

data/spec/background_queue/server_lib/integration/queue_integration_spec.rb

@@ -88,9 +88,9 @@ describe "Queue Integration" do
       
       task2.get_job.total_tasks.should eq(2)
 
-      subject.next_item.should be_nil
-      subject.finish_item(task)
       subject.next_item.id.should eq(:task_id2)
+      subject.finish_item(task)
+      #subject.next_item.id.should eq(:task_id2)
       
     end
   end
@@ -103,6 +103,7 @@ describe "Queue Integration" do
       subject.add_task(SimpleTask.new(:owner_id, :job_id, :task_id, 3, {:synchronous=>true}))
       task = subject.next_item
       task.id.should be(:task_id)
+      subject.finish_item(task)
       subject.next_item.should be_nil 
     end
     
@@ -110,10 +111,12 @@ describe "Queue Integration" do
       subject.add_task(SimpleTask.new(:owner_id, :job_id, :task_id, 3, {:synchronous=>true}))
       task = subject.next_item
       task.id.should be(:task_id)
+      task.running = true
       subject.next_item.should be_nil 
       subject.add_task(SimpleTask.new(:owner_id, :job_id, :task_id, 3, {:synchronous=>true}))
       subject.next_item.should be_nil 
-      subject.finish_item(task)
+      task.running = false
+      subject.finish_task(task)
       task = subject.next_item
       task.id.should be(:task_id)
     end

data/spec/background_queue/server_lib/job_spec.rb

@@ -9,14 +9,14 @@ describe BackgroundQueue::ServerLib::Job do
   
   it "#add_item uses normal priority queue" do
     task.should_receive(:set_job).with(subject)
-    task.should_receive(:synchronous?).and_return(false)
+    #task.should_receive(:synchronous?).and_return(false)
     BackgroundQueue::ServerLib::Job.any_instance.should_receive(:push).with(task).and_return(nil)
     subject.add_item(task)
   end
   
   it "#next_item uses normal priority queue" do
-    BackgroundQueue::ServerLib::Job.any_instance.should_receive(:pop).and_return(:task)
-    subject.next_item.should eq(:task)
+    BackgroundQueue::ServerLib::Job.any_instance.should_receive(:pop).and_return(task)
+    subject.next_item.should eq(task)
   end
   
   context "#add_item" do
@@ -320,6 +320,47 @@ describe BackgroundQueue::ServerLib::Job do
     end
   end
   
+  context "#update_summary_meta" do
+    it "will append data" do
+      subject.update_summary_meta({:summary=>"app", :type=>"test", :data=>1})
+      subject.update_summary_meta({:summary=>"app", :type=>"test", :data=>2})
+      subject.summary[:test].should eq([1,2])
+    end
+    
+    it "will set data" do
+      subject.update_summary_meta({:summary=>"set", :type=>"test", :key=>'a', :data=>1})
+      subject.update_summary_meta({:summary=>"set", :type=>"test", :key=>'b', :data=>2})
+      subject.summary[:test].should eq({'a'=>1, 'b'=>2})
+    end
+    
+    it "will increment data" do
+      subject.update_summary_meta({:summary=>"inc", :type=>"test", :data=>1})
+      subject.summary[:test].should eq(1)
+      subject.update_summary_meta({:summary=>"inc", :type=>"test", :data=>2})
+      subject.summary[:test].should eq(3)
+    end
+    
+    it "will decrement data" do
+      subject.update_summary_meta({:summary=>"inc", :type=>"test", :data=>4})
+      subject.summary[:test].should eq(4)
+      subject.update_summary_meta({:summary=>"dec", :type=>"test", :data=>2})
+      subject.summary[:test].should eq(2)
+    end
+    
+    it "will reset data" do
+      subject.update_summary_meta({:summary=>"inc", :type=>"test", :data=>4})
+      subject.summary[:test].should eq(4)
+      subject.update_summary_meta({:summary=>"res", :type=>"test"})
+      subject.summary[:test].should be_nil
+      subject.update_summary_meta({:summary=>"inc", :type=>"test", :data=>4})
+      subject.update_summary_meta({:summary=>"inc", :type=>"test2", :data=>4})
+      subject.summary[:test].should eq(4)
+      subject.summary[:test2].should eq(4)
+      subject.update_summary_meta({:summary=>"res", :type=>"all"})
+      subject.summary.should eq({})
+    end
+  end
+  
   context "integration" do
     it "will track progress" do
       task0 = BackgroundQueue::ServerLib::Task.new(:owner_id, :job_id, :id0, 1, :worker, {}, {:exclude=>true})

data/spec/background_queue/server_lib/worker_thread_spec.rb

@@ -64,7 +64,7 @@ describe BackgroundQueue::ServerLib::WorkerThread do
       server.stub('running?'=>true)
       worker = double("worker")
       task = DefaultTask.new
-      server.error_tasks.should_receive(:add_task).with(task)
+      server.task_queue.should_receive(:add_task_to_error_list).with(task)
       worker_client = double("worker_client")
       subject.should_receive(:build_client).and_return(worker_client)
       worker_client.should_receive(:send_request).with(worker, task, :secret).and_return(:fatal_error)
@@ -77,7 +77,7 @@ describe BackgroundQueue::ServerLib::WorkerThread do
       server.stub('running?'=>true)
       worker = double("worker")
       task = DefaultTask.new
-      server.error_tasks.should_receive(:add_task).with(task)
+      server.task_queue.should_receive(:add_task_to_error_list).with(task)
       worker_client = double("worker_client")
       subject.should_receive(:build_client).and_return(worker_client)
       worker_client.should_receive(:send_request).with(worker, task, :secret).and_return(:worker_error)

data/spec/background_queue/utils_spec.rb

@@ -23,5 +23,35 @@ describe "Utils" do
       any_hash["a"].should eq(:b)
     end
   end
+  
+  
+   context "#current_environment" do
+      it "gets_the_current environment from env" do
+        ENV.should_receive(:has_key?).with("RAILS_ENV") { true }
+        ENV.should_receive(:[]).with("RAILS_ENV") { :the_env }
+        BackgroundQueue::Utils.current_environment.should eq(:the_env)
+      end
+      
+      it "gets_the_current environment from Rails" do
+        ENV.should_receive(:has_key?).with("RAILS_ENV") { false }
+        Rails.should_receive(:env) { :the_env }
+        BackgroundQueue::Utils.current_environment.should eq(:the_env)
+      end
+      
+      context "with development environment" do
+        before do
+          BackgroundQueue::Config.stub(:current_environment) { 'development' }
+        end
+        
+        it "extracts the correct environment entry from the hash" do
+          BackgroundQueue::Config.__prv__extract_enviroment_entry({:development=>:test}, :path_that_exists).should eq(:test)
+        end
+        
+        it "errors if the YAML does not define environment entry" do
+          File.stub(:expand_path) { :expanded_path }
+          expect { BackgroundQueue::Config.__prv__extract_enviroment_entry({:test=>:test}, :path_that_exists).should eq(:test)}.to raise_error(BackgroundQueue::LoadError, "Error loading YAML for background_queue configuration file at 'expanded_path': missing enviroment root entry: development")
+        end
+      end
+    end
 
 end

data/spec/background_queue/worker/calling_spec.rb

@@ -87,9 +87,11 @@ describe "Calling" do
   
   context "#call_worker" do
     it "will call worker.run within a render block" do
-      env = double("env")
+      env = double("env", :step=>nil)
       worker = double("worker")
       worker.should_receive(:set_environment).with(nil)
+      subject.should_receive(:set_process_name).with(env)
+      subject.should_receive(:revert_process_name)
       env.should_receive(:set_output).with(:output)
       worker.should_receive(:run)
       subject.should_receive(:render) { |opts|

data/spec/background_queue/worker/environment_spec.rb

@@ -17,7 +17,8 @@ describe BackgroundQueue::Worker::Environment do
     {
       :auth=>"ABCD",
       :task=>{:params=>{:a=>"b"}, :owner_id=>'oid', :job_id=>'jid', :id=>'tid', :priority=>1}.to_json,
-      :server_port=>213
+      :server_port=>213,
+      :step=>"start"
     }
   }
   
@@ -42,6 +43,7 @@ describe BackgroundQueue::Worker::Environment do
       subject.job_id.should eq("jid")
       subject.task_id.should eq("tid")
       subject.priority.should eq(1)
+      subject.step.should eq("start")
     end
     
     it "will error if the task definition is not valid json" do

data/spec/background_queue/worker/logger_spec.rb

@@ -0,0 +1,58 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue_worker'
+
+
+module ActiveRecord
+  class Base
+    @@logger = :ar_logger
+    
+    def self.logger
+      @@logger
+    end
+    
+    def self.logger=(other)
+      @@logger = other
+    end
+  end
+end
+
+describe BackgroundQueue::Worker::Logger do
+  
+  let(:faux_logger) {
+    double("logger")
+  }
+
+  context "#build_logger" do
+    it "will init a logger object" do
+      logger = BackgroundQueue::Worker::Logger.build_logger("/tmp/bgq_test1.log", "task_key", Logger::DEBUG)
+    end
+  end
+  
+  context "#init_logger" do
+    it "will reset the system loggers" do
+      BackgroundQueue::Utils.should_receive(:current_root).and_return("ROOT")
+      BackgroundQueue::Worker::Logger.should_receive(:build_logger).with("ROOT/log/workers/worker_name-owner_key-job_key.log", "task_key", :level).and_return(faux_logger)
+      
+      faux_logger.should_receive(:set_previous_state).with({:ar_base=>:ar_logger, :rails_default_logger=>:abc})
+      
+      RAILS_DEFAULT_LOGGER = :abc
+      
+      logger = BackgroundQueue::Worker::Logger.init_logger("worker_name", "owner_key", "job_key", "task_key", :level)
+      
+      RAILS_DEFAULT_LOGGER.should be(logger)
+      ActiveRecord::Base.logger.should be(logger)
+      
+    end
+  end
+  
+  context "#revert_to_previous_state" do
+    it "will set the state back" do
+      logger = BackgroundQueue::Worker::Logger.new("/tmp/bgq_test1.log", "task_key")
+      logger.set_previous_state({:ar_base=>:ar_logger1, :rails_default_logger=>:abc1})
+      logger.revert_to_previous_state
+      RAILS_DEFAULT_LOGGER.should be(:abc1)
+      ActiveRecord::Base.logger.should be(:ar_logger1)
+    end
+  end
+    
+end

data/spec/background_queue/worker/progress_spec.rb

@@ -0,0 +1,82 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue_worker'
+
+
+describe BackgroundQueue::Worker::Progress do
+  
+  let(:callback_object) {
+    double("callback_object")
+  }
+  
+  subject { BackgroundQueue::Worker::Progress.new(callback_object) }
+  
+  context "#start" do
+    it "will start at zero with the caption" do
+      callback_object.should_receive(:set_progress).with("test", 0.0, subject)
+      subject.start("test")
+    end
+    
+    it "will start at zero with no caption" do
+      callback_object.should_receive(:set_progress).with("", 0.0, subject)
+      subject.start("")
+    end
+  end
+  
+  context "#set_main_caption" do
+    it "will update the main caption" do
+      callback_object.should_receive(:set_progress).with("test", 0.0, subject)
+      callback_object.should_receive(:set_progress).with("test2", 0.0, subject)
+      subject.start("test")
+      subject.set_main_caption("test2")
+    end
+  end
+  
+  context "#finish" do
+    it "will set the progress to 100% and reset the caption" do
+      callback_object.should_receive(:set_progress).with("test", 100.0, subject)
+      subject.finish("test")
+    end
+  end
+  
+  context "#register_task" do
+    it "will store the weight against the key" do
+      subject.register_task(:key, 10)
+      subject.registered_tasks[:key].should eq(10)
+    end
+  end
+  
+  context "#get_task_size" do
+    it "will get the size of the task" do
+      subject.register_task(:key, 10)
+      subject.register_task(:key2, 10)
+      subject.get_task_size(:key).should eq(50)
+    end
+    
+  end
+  
+  context "#start_task" do
+    it "will finish any existing task" do
+      callback_object.stub(:set_progress)
+      subject.register_task(:key, 10)
+      subject.register_task(:key2, 10)
+      subject.start("main")
+      subject.start_task(:key,"sub")
+      subject.get_percent.should eq(0)
+      subject.get_caption.should eq("main: sub")
+      subject.start_task(:key2, "sub2")
+      subject.get_percent.should eq(50)
+      subject.get_caption.should eq("main: sub2")
+    end
+  end
+  
+  context "#set_task_steps" do
+    it "will calculate the step size" do
+      callback_object.stub(:set_progress)
+      subject.register_task(:key, 10)
+      subject.start_task(:key,"sub")
+      subject.set_task_steps(50)
+      subject.inc
+      subject.get_percent.should eq(2)
+    end
+  end
+end

data/spec/background_queue/worker/worker_loader_spec.rb

@@ -72,7 +72,7 @@ describe BackgroundQueue::Worker::WorkerLoader do
     it "will reload the worker if the filestamp has changed" do
       File.should_receive(:mtime).with(:path).and_return(Time.at(11))
       subject.should_receive(:load_file).with(:path).and_return(true)
-      subject.should_receive(:load_class).with(:worker_name).and_return(:new_worker)
+      subject.should_receive(:load_class).with(:worker_name, :path).and_return(:new_worker)
       
       subject.reload_if_updated(worker_entry)
       worker_entry.worker.should eq(:new_worker)

data/spec/resources/summary_worker.rb

@@ -0,0 +1,18 @@
+class SummaryWorker < BackgroundQueue::Worker::Base
+
+  def run
+    begin
+      if params[:mode] == "summary"
+        raise "Invalid Summary (#{summary.inspect})" if summary[:test] != [1, 2] && summary[:test] != [2, 1]
+      else
+        raise "Missing TestId" if params[:test_id].nil?
+        add_progress_meta(:meta, {:test_meta=>"something"})
+        append_summary(:test, params[:test_id])
+      end
+      set_progress("Done", 100)
+    rescue Exception=>e
+      puts e.message
+      puts e.backtrace.join("\n")
+    end
+  end
+end

data/spec/shared/queue_registry_shared.rb

@@ -99,13 +99,14 @@ shared_examples "a queue registry" do
         bg.add_item(task2)
         bg
       }
+      
         
       it "will add to stalled items if no items left for queue" do
-        subject.should_receive(:remove_item_from_queue).with(any_args) { [true, 2, :task]}
+        subject.should_receive(:remove_item_from_queue).with(any_args) { [true, 2, task1]}
         queue_class.any_instance.stub(:empty?) { true }
         subject.should_receive(:stall_queue)
         subject.__prv__get_queue_for_priority(2, false)
-        subject.next_item.should eq(:task)
+        subject.next_item.should eq(task1)
         subject.__prv__get_queues.should have(1).items
         subject.__prv__get_queues.first.priority.should eq(3)
         #make sure the owner was removed
@@ -115,7 +116,7 @@ shared_examples "a queue registry" do
       
       
       it "will lower priority when items left are lower priority" do
-        subject.should_receive(:remove_item_from_queue).with(any_args) { [true, 2, :task]}
+        subject.should_receive(:remove_item_from_queue).with(any_args) { [true, 2, task1]}
         queue_class.any_instance.stub(:empty?) { false }
         
         subject.__prv__get_queue(:owner_id2, true).last.set_priority(3)

data/spec/support/simple_task.rb

@@ -55,4 +55,28 @@ class SimpleTask
   def initial_progress_caption
     @options[:initial_progress_caption]
   end
+  
+  def send_summary?
+    @options[:send_summary]
+  end
+  
+  def step
+    @options[:step]
+  end
+  
+  def set_error_status(e_status)
+    @error_status = e_status
+  end
+  
+  def get_error_status
+    @error_status
+  end
+  
+  def waiting_to_retry?
+    @error_status == :waiting_to_retry
+  end
+  
+  def replaced_while_waiting_to_retry?
+    @error_status == :replaced_while_waiting_to_retry
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: background_queue
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.6.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-17 00:00:00.000000000 Z
+date: 2012-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &19238880 !ruby/object:Gem::Requirement
+  requirement: &4758140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *19238880
+  version_requirements: *4758140
 - !ruby/object:Gem::Dependency
   name: rufus-scheduler
-  requirement: &20184220 !ruby/object:Gem::Requirement
+  requirement: &4769420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *20184220
+  version_requirements: *4769420
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &20182860 !ruby/object:Gem::Requirement
+  requirement: &5013180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.12.10
   type: :runtime
   prerelease: false
-  version_requirements: *20182860
+  version_requirements: *5013180
 - !ruby/object:Gem::Dependency
   name: ipaddress
-  requirement: &20180600 !ruby/object:Gem::Requirement
+  requirement: &5020560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 0.8.0
   type: :runtime
   prerelease: false
-  version_requirements: *20180600
+  version_requirements: *5020560
 - !ruby/object:Gem::Dependency
   name: ipaddress
-  requirement: &20178440 !ruby/object:Gem::Requirement
+  requirement: &5033300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.8.0
   type: :runtime
   prerelease: false
-  version_requirements: *20178440
+  version_requirements: *5033300
 - !ruby/object:Gem::Dependency
   name: algorithms
-  requirement: &20177140 !ruby/object:Gem::Requirement
+  requirement: &5030560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +76,10 @@ dependencies:
         version: 0.5.0
   type: :runtime
   prerelease: false
-  version_requirements: *20177140
+  version_requirements: *5030560
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &19623560 !ruby/object:Gem::Requirement
+  requirement: &5028420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: 2.9.0
   type: :development
   prerelease: false
-  version_requirements: *19623560
+  version_requirements: *5028420
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &19621540 !ruby/object:Gem::Requirement
+  requirement: &5027080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -98,10 +98,10 @@ dependencies:
         version: 1.8.3
   type: :development
   prerelease: false
-  version_requirements: *19621540
+  version_requirements: *5027080
 - !ruby/object:Gem::Dependency
   name: yard
-  requirement: &19617840 !ruby/object:Gem::Requirement
+  requirement: &5042180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -109,10 +109,10 @@ dependencies:
         version: '0.7'
   type: :development
   prerelease: false
-  version_requirements: *19617840
+  version_requirements: *5042180
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &19615920 !ruby/object:Gem::Requirement
+  requirement: &5041260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -120,10 +120,10 @@ dependencies:
         version: '3.12'
   type: :development
   prerelease: false
-  version_requirements: *19615920
+  version_requirements: *5041260
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &20601320 !ruby/object:Gem::Requirement
+  requirement: &5038300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -131,10 +131,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *20601320
+  version_requirements: *5038300
 - !ruby/object:Gem::Dependency
   name: redcarpet
-  requirement: &20599700 !ruby/object:Gem::Requirement
+  requirement: &5036560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -142,7 +142,7 @@ dependencies:
         version: 2.1.1
   type: :development
   prerelease: false
-  version_requirements: *20599700
+  version_requirements: *5036560
 description: Organise background tasks so they will not overload the machine(s) running
   the tasks, while still giving a fair, balanced allocation of running time to members
   in the queue
@@ -199,6 +199,8 @@ files:
 - lib/background_queue/worker/calling.rb
 - lib/background_queue/worker/config.rb
 - lib/background_queue/worker/environment.rb
+- lib/background_queue/worker/logger.rb
+- lib/background_queue/worker/progress.rb
 - lib/background_queue/worker/worker_loader.rb
 - lib/background_queue_server.rb
 - lib/background_queue_worker.rb
@@ -213,6 +215,7 @@ files:
 - spec/background_queue/server_lib/error_task_list_spec.rb
 - spec/background_queue/server_lib/event_connection_spec.rb
 - spec/background_queue/server_lib/event_server_spec.rb
+- spec/background_queue/server_lib/integration/error_handling_spec.rb
 - spec/background_queue/server_lib/integration/full_test_spec.rb
 - spec/background_queue/server_lib/integration/queue_integration_spec.rb
 - spec/background_queue/server_lib/integration/serialize_spec.rb
@@ -232,12 +235,15 @@ files:
 - spec/background_queue/worker/base_spec.rb
 - spec/background_queue/worker/calling_spec.rb
 - spec/background_queue/worker/environment_spec.rb
+- spec/background_queue/worker/logger_spec.rb
+- spec/background_queue/worker/progress_spec.rb
 - spec/background_queue/worker/worker_loader_spec.rb
 - spec/resources/config-client.yml
 - spec/resources/config-serialize.yml
 - spec/resources/config.yml
 - spec/resources/example_worker.rb
 - spec/resources/example_worker_with_error.rb
+- spec/resources/summary_worker.rb
 - spec/resources/test_worker.rb
 - spec/shared/queue_registry_shared.rb
 - spec/spec_helper.rb
@@ -261,7 +267,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 613144620285186166
+      hash: 1524954307267393348
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: