background_queue 0.2.0

data/spec/background_queue/server_lib/worker_client_spec.rb

@@ -0,0 +1,196 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue_server'
+
+
+describe BackgroundQueue::ServerLib::WorkerClient do
+
+  subject { BackgroundQueue::ServerLib::WorkerClient.new(SimpleServer.new({:config=>double("cfg", {:address=>double("addr", {:port=>123})} )})) }
+  
+  
+  context "#build_request" do
+    it "builds a request payload from a task" do
+      task = SimpleTask.new(:owner_id, :job_id, :task_id, 3, { :domain=>"www.example.com" })
+      task.should_receive(:to_json).and_return(:the_json)
+      Net::HTTP::Post.any_instance.should_receive(:set_form_data).with({:task=>:the_json, :auth=>"auth", :server_port=>123})
+      Net::HTTP::Post.any_instance.should_receive("[]=".intern).with("host", "www.example.com")
+      Net::HTTP::Post.any_instance.should_receive("[]=".intern).with(any_args).at_least(:once)
+      request = subject.__prv__build_request(URI("http://127.0.0.1:3000/worker/run"), task, "auth")
+    end
+  end
+  
+  context "#send_request" do
+    it "uses http client to call the worker" do
+      uri = URI("http://127.0.0.1:3000/worker/run")
+      worker_config = BackgroundQueue::ServerLib::Worker.new(uri)
+      task = SimpleTask.new(:owner_id, :job_id, :task_id, 3, { :domain=>"www.example.com" })
+      subject.should_receive(:build_request).with(uri, task, "auth").and_return(:post_request)
+      
+      http_instance = double("http")
+      http_instance.should_receive(:request).with(:post_request).and_yield(:http_response)
+      subject.should_receive(:read_response).with(worker_config, :http_response, task).and_return(true)
+      
+      Net::HTTP.should_receive(:start).with("127.0.0.1", 3000).and_yield(http_instance)
+      subject.send_request(worker_config, task, "auth").should be_true
+      
+    end
+    
+    it "will fail if the connection fails" do
+      uri = URI("http://127.0.0.1:3000/worker/run")
+      worker_config = BackgroundQueue::ServerLib::Worker.new(uri)
+      task = SimpleTask.new(:owner_id, :job_id, :task_id, 3, { :domain=>"www.example.com" })
+      subject.should_receive(:build_request).with(uri, task, "auth").and_return(:post_request)
+      Net::HTTP.should_receive(:start).with("127.0.0.1", 3000).and_raise("connection error")
+      subject.send_request(worker_config, task, "auth").should be_false
+    end
+  end
+  
+  context "#read_response" do
+    it "will read the streamed response" do
+      http_response = double("response", :code=>"200")
+      http_response.should_receive(:read_body).and_yield("data")
+      subject.should_receive(:process_chunk).with("data", :task)
+      subject.__prv__read_response(:worker_config, http_response, :task)
+    end
+    
+    it "will raise an error if response is not 200" do
+      http_response = double("response", :code=>"400")
+      worker_config = double("wc", :url=>"uri")
+      expect { subject.__prv__read_response(worker_config, http_response, :task) }.to raise_exception
+    end
+  end
+  
+  context "#process_response_chunk" do
+    it "will handle a reponse in 1 chunk" do
+      subject.should_receive(:process_line).with("data", :task)
+      subject.__prv__process_chunk("data\n", :task)
+    end
+    
+    it "will handle a partial response" do
+      subject.should_not_receive(:process_line)
+      subject.__prv__process_chunk("data", :task)
+    end
+    
+    it "will handle multiple responses in 1 chunk" do
+      subject.should_receive(:process_line).with("data", :task).once
+      subject.should_receive(:process_line).with("data2", :task).once
+      subject.__prv__process_chunk("data\ndata2\n", :task)
+    end
+    
+    it "will handle multiple partial responses" do
+      subject.should_receive(:process_line).with("data", :task).once
+      subject.should_receive(:process_line).with("data2", :task).once
+      subject.should_receive(:process_line).with("data3", :task).once
+      subject.should_receive(:process_line).with("data4", :task).once
+      subject.__prv__process_chunk("da", :task)
+      subject.__prv__process_chunk("ta\ndata2\nda", :task)
+      subject.__prv__process_chunk("ta3\nda", :task)
+      subject.__prv__process_chunk("ta4\n", :task)
+    end
+  end
+  
+  context "#process_line" do
+    it "loads from json" do
+      subject.should_receive(:set_worker_status).with({'a'=>'b'}, :task)
+      subject.__prv__process_line('{ "a":"b"}', :task).should be_true
+    end
+    
+    it "fails gracefully on invalid json" do
+      subject.__prv__process_line('{ "a":"b"', DefaultTask.new).should be_false
+    end
+  end
+  
+  context "#set_worker_status" do
+    it "calls set_worker_status on the task" do
+      task = DefaultTask.new
+      
+      BackgroundQueue::Utils::AnyKeyHash.should_receive(:new).with(:status).and_return(:kstatus)
+      task.should_receive(:set_worker_status).with(:kstatus)
+      subject.__prv__set_worker_status(:status, task)
+    end
+  end
+  
+  
+  context "can handle thread cancelling" do
+    #this can cause issues with other tests....
+    xit "will return false if exception raised" do
+      
+      mutex = Mutex.new
+      resource = ConditionVariable.new
+      
+      mutex2 = Mutex.new
+      resource2 = ConditionVariable.new
+
+      run_request = false
+      ss = TestWorkerServer.new(8001)
+      ss.start(Proc.new { |controller|
+        #puts "in proc"
+          
+        mutex2.synchronize {
+          resource2.signal
+        }
+        #puts "in proc: waiting" 
+        mutex.synchronize {
+          resource.wait(mutex)
+        }
+        #puts "waited"
+        run_request = true
+        controller.render :text =>{:percent=>100, :caption=>"Done"}.to_json, :type=>"text/text"
+      })
+      
+      
+      uri = URI("http://127.0.0.1:8001/background_queue")
+      worker_config = BackgroundQueue::ServerLib::Worker.new(uri)
+      task = SimpleTask.new(:owner_id, :job_id, :task_id, 3, { :domain=>"www.example.com" })
+      
+      call_result = nil
+      t1 = Thread.new {
+        #puts "calling"
+        begin
+          status = Timeout::timeout(2) {
+            call_result = subject.send_request(worker_config, task, "abcd")
+           # puts "called"
+          }
+        rescue Timeout::Error=>te
+          #puts "timeout"
+          call_result = :timeout
+        end
+        mutex2.synchronize {
+          resource2.signal
+        }
+      }
+
+      #wait until we know the request has been sent and is processing
+      mutex2.synchronize {
+        resource2.wait(mutex2)
+      }
+      run_request.should be_false
+      #puts "cancelling"
+      t1.raise "Cancelling From Thread"
+      
+      #puts "canceled"
+      #wait until we know the request has been cancelled
+      mutex2.synchronize {
+        resource2.wait(mutex2)
+      }
+      call_result.should be_false
+      run_request.should be_false
+
+      mutex.synchronize {
+        resource.signal
+      }
+     
+      
+      t1.join
+      ss.stop
+      
+    end
+    
+    it "will do nothing if not started" do
+      
+    end
+    
+  end
+  
+  
+  
+end

data/spec/background_queue/server_lib/worker_thread_spec.rb

@@ -0,0 +1,125 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue_server'
+
+
+describe BackgroundQueue::ServerLib::WorkerThread do
+  
+  let(:balanced_queue) {
+    double("balanced_queue")
+  }
+  
+  let(:workers) {
+    double("workers")
+  }
+  
+  let(:server) {
+    SimpleServer.new(:task_queue=>balanced_queue, :workers=>workers, :config=>double("conf", :secret=>:secret))
+  }
+  
+  subject { BackgroundQueue::ServerLib::WorkerThread.new(server) }
+  
+  context "#get_next_task" do
+    it "will get the next task from the servers balanced queue" do
+      server.task_queue.should_receive(:next_task).and_return(:task)
+      server.stub('running?'=>true)
+      subject.get_next_task.should eq(:task)
+    end
+    
+    it "will try again if the balanced queue returns nil" do
+      count = 0
+      server.task_queue.should_receive(:next_task).twice { 
+        count += 1
+        count == 2 ? :task : nil
+      }
+      server.stub('running?'=>true)
+      subject.get_next_task.should eq(:task)
+    end
+    
+    it "will stop if the server has stopped" do
+      server.should_receive(:running?).and_return(false)
+      subject.get_next_task.should be_nil
+    end
+    
+  end
+  
+  
+  context "#call_worker" do
+    it "will get a worker and call it" do
+      server.stub('running?'=>true)
+      worker = double("worker")
+      task = DefaultTask.new
+      BackgroundQueue::ServerLib::WorkerClient.any_instance.should_receive(:send_request).with(worker, task, :secret).and_return(true)
+      server.workers.should_receive(:get_next_worker).and_return(worker)
+      server.workers.should_receive(:finish_using_worker).with(worker, true)
+      server.task_queue.should_receive(:finish_task).with(task)
+      subject.call_worker(task).should be_true
+    end
+    
+    it "will keep trying if the worker fails" do
+      server.stub('running?'=>true)
+      worker = double("worker")
+      task = DefaultTask.new
+      count = 0
+      worker_client1 = double("w1")
+      worker_client1.should_receive(:send_request).with(worker, task, :secret).and_return(false)
+      worker_client2 = double("w2")
+      worker_client2.should_receive(:send_request).with(worker, task, :secret).and_return(true)
+      
+      subject.should_receive(:build_client).twice {
+        count += 1
+        count == 1 ? worker_client1 : worker_client2
+      }
+      server.workers.should_receive(:get_next_worker).twice.and_return(worker)
+      server.workers.should_receive(:finish_using_worker).with(worker, false)
+      server.workers.should_receive(:finish_using_worker).with(worker, true)
+      server.task_queue.should_receive(:finish_task).with(task)
+      subject.call_worker(task).should be_true
+    end
+    
+    it "will sleep and try again if there are no workers" do
+      server.stub('running?'=>true)
+      count = 0
+      task = DefaultTask.new
+      worker = double("worker")
+      server.workers.should_receive(:get_next_worker).twice {
+        count += 1
+        count == 1 ? nil : worker
+      }
+      Kernel.should_receive(:sleep).with(1)
+      BackgroundQueue::ServerLib::WorkerClient.any_instance.should_receive(:send_request).with(worker, task, :secret).and_return(true)
+      server.workers.should_receive(:finish_using_worker).with(worker, true)
+      server.task_queue.should_receive(:finish_task).with(task)
+      subject.call_worker(task).should be_true
+    end
+    
+    it "will stop trying if the server is not running" do
+      server.stub('running?'=>false)
+      task = DefaultTask.new
+      server.task_queue.should_receive(:finish_task).with(task)
+      server.task_queue.should_receive(:add_task).with(task)
+      subject.call_worker(task).should be_false
+    end
+  end
+  
+  context "#run" do
+    before do
+      count = 0
+      server.should_receive('running?').twice {
+        count += 1
+        count < 2
+      }  
+    end
+    
+    it "will keep running while the server is running" do
+      subject.should_receive(:get_next_task).and_return(:task)
+      subject.should_receive(:call_worker).with(:task)
+      subject.run
+    end
+    
+    it "will not call_worker if no task" do
+      subject.should_receive(:get_next_task).and_return(nil)
+      subject.run
+    end
+  end
+  
+end

data/spec/background_queue/utils_spec.rb

@@ -0,0 +1,27 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'background_queue'
+
+describe "Utils" do
+
+  context "hash" do
+    it "gets an entry by string" do
+      BackgroundQueue::Utils.get_hash_entry({:key=>:value}, "key").should eq(:value)
+      BackgroundQueue::Utils.get_hash_entry({'key'=>:value}, "key").should eq(:value)
+    end
+    
+    it "gets an entry by symbol" do
+      BackgroundQueue::Utils.get_hash_entry({:key=>:value}, :key).should eq(:value)
+      BackgroundQueue::Utils.get_hash_entry({'key'=>:value}, :key).should eq(:value)
+    end
+  end
+  
+  context "any key hash" do
+    it "wraps hash with any key accessor" do
+      hash = {:a=>:b}
+      any_hash = BackgroundQueue::Utils::AnyKeyHash.new(hash)
+      any_hash[:a].should eq(:b)
+      any_hash["a"].should eq(:b)
+    end
+  end
+
+end

data/spec/background_queue/worker/base_spec.rb

@@ -0,0 +1,35 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue_worker'
+
+
+describe BackgroundQueue::Worker::Base do
+
+  let(:logger) {
+    l = double("logger")
+    l.stub(:debug)
+    l.stub(:info)
+    l.stub(:error)
+    l.stub(:warn)
+    l
+  }
+
+  let(:env) {
+    double("env", :logger=>logger)
+  }
+  
+  context "#set_progress" do
+    it "will serialize the progress" do
+      subject.set_environment(env)
+      env.should_receive(:send_data).with({:caption=>"TEST", :percent=>15.1}.to_json)
+      subject.set_progress("TEST", 15.1)
+    end
+  end
+  
+  context "#add_progress_meta" do
+    it "will serialize the meta" do
+      subject.set_environment(env)
+      env.should_receive(:send_data).with({:meta=>{:a=>"b"}}.to_json)
+      subject.add_progress_meta(:a, 'b')
+    end
+  end
+end

data/spec/background_queue/worker/calling_spec.rb

@@ -0,0 +1,103 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+require 'background_queue_worker'
+
+class DummyClass
+  include BackgroundQueue::Worker::Calling
+end
+
+describe "Calling" do
+
+  let(:logger) {
+    l = double("logger")
+    l.stub(:debug)
+    l.stub(:info)
+    l.stub(:error)
+    l.stub(:warn)
+    l
+  }
+  
+  let(:params) {
+    {
+      :auth=>"ABCD"
+    }
+  }
+  
+  let(:response) {
+    double("resp")
+  }
+  
+  let(:headers) { {} }
+  
+  
+  let(:subject) {
+    s = DummyClass.new
+    s.stub(:params).and_return(params)
+    s.stub(:logger).and_return(logger)
+    s.stub(:response).and_return(response)
+    s.stub(:headers).and_return(headers)
+    s
+  }
+  
+  before do
+    BackgroundQueue::Worker::Config.secret = "ABCD"
+  end
+  
+  context "#check_secret" do
+    it "will pass if the secret matches" do
+      subject.check_secret.should be_true
+    end
+    
+    it "will fail and return a 401 auth error if secret does not match" do
+      BackgroundQueue::Worker::Config.secret = "XYZ"
+      subject.should_receive(:render).with(:text=>"Invalid auth (ABCD)", :status=>401)
+      subject.check_secret.should be_false
+    end
+  end
+  
+  context "#init_environment" do
+    it "will instanciate an environment and init it" do
+      BackgroundQueue::Worker::Environment.any_instance.should_receive(:init_from_controller).with(subject)
+      subject.init_environment
+    end
+  end
+  
+  context "#run_worker" do
+    it "should return if check_secret fails" do
+      subject.should_receive(:check_secret).and_return(false)
+      subject.run_worker()
+    end
+    
+    it "will call worker if all is ok" do
+      env = double("env", :worker=>:worker_name)
+      worker = double("worker")
+      worker.should_receive(:set_environment).with(env)
+      
+      subject.should_receive(:call_worker).with(worker, env)
+      subject.should_receive(:init_environment).and_return(env)
+      BackgroundQueue::Worker::WorkerLoader.should_receive(:get_worker).with(:worker_name).and_return(worker)
+      subject.run_worker
+    end
+    
+    it "will render the error and raise it if worker does not initialize correctly" do
+      subject.should_receive(:init_environment).and_raise "ERROR"
+      subject.should_receive(:render).with(:text=>"Error initializing worker: ERROR", :status=>500)
+      expect { subject.run_worker }.to raise_exception("ERROR")
+    end
+  end
+  
+  context "#call_worker" do
+    it "will call worker.run within a render block" do
+      env = double("env")
+      worker = double("worker")
+      worker.should_receive(:set_environment).with(nil)
+      env.should_receive(:set_output).with(:output)
+      worker.should_receive(:run)
+      subject.should_receive(:render) { |opts|
+        opts[:text].call(:response, :output)
+      }
+      subject.call_worker(worker, env)
+      subject.headers['X-Accel-Buffering'].should eq('no')
+    end
+  end
+
+end