chore-core 1.8.2 → 3.2.3

data/spec/chore/queues/sqs/consumer_spec.rb

@@ -34,9 +34,7 @@ describe Chore::Queues::SQS::Consumer do
 
       expect(AWS::SQS).to receive(:new).with(
         :access_key_id => 'key',
-        :secret_access_key => 'secret',
-        :logger => Chore.logger,
-        :log_level => :debug
+        :secret_access_key => 'secret'
       ).and_return(sqs)
       consumer.consume
     end

data/spec/chore/strategies/consumer/batcher_spec.rb

@@ -90,4 +90,54 @@ describe Chore::Strategy::Batcher do
       subject.batch.should == ['test']
     end
   end
+
+  describe 'schedule' do
+    let(:timeout) { 5 }
+    let(:batch) { [] }
+
+    before(:each) do
+      Thread.stub(:new) do |&block|
+        # Stop the batcher on the next iteration
+        subject.stub(:sleep) { subject.stop }
+
+        # Run the scheduling thread
+        block.call(timeout)
+      end
+
+      subject.batch = batch.dup
+    end
+
+    context 'with no items' do
+      it 'should not invoke the callback' do
+        callback.should_not_receive(:call)
+        subject.schedule(timeout)
+      end
+    end
+
+    context 'with new items' do
+      let(:batch) do
+        [
+          Chore::UnitOfWork.new.tap {|work| work.created_at = Time.now - 2}
+        ]
+      end
+
+      it 'should not invoke the callback' do
+        callback.should_not_receive(:call).with(batch)
+        subject.schedule(timeout)
+      end
+    end
+
+    context 'with old items' do
+      let(:batch) do
+        [
+          Chore::UnitOfWork.new.tap {|work| work.created_at = Time.now - 6}
+        ]
+      end
+
+      it 'should invoke the callback' do
+        callback.should_receive(:call).with(batch)
+        subject.schedule(timeout)
+      end
+    end
+  end
 end

data/spec/chore/strategies/consumer/threaded_consumer_strategy_spec.rb

@@ -51,6 +51,7 @@ describe Chore::Strategy::ThreadedConsumerStrategy do
       work.message.should == "test"
       work.previous_attempts.should == 0
       work.current_attempt.should == 1
+      work.created_at.should_not be_nil
     end
   end
 

data/spec/chore/strategies/consumer/throttled_consumer_strategy_spec.rb

@@ -0,0 +1,165 @@
+require 'spec_helper'
+
+class TestConsumer < Chore::Consumer
+  def initialize(queue_name, opts={})
+  end
+
+  def consume
+    # just something that looks like an SQS message
+    msg = OpenStruct.new( :id => 1, :body => "test" )
+    yield msg if block_given?
+  end
+end
+
+class NoQueueConsumer < Chore::Consumer
+  def initialize(queue_name, opts={})
+    raise Chore::TerribleMistake
+  end
+
+  def consume
+  end
+end
+
+describe Chore::Strategy::ThrottledConsumerStrategy do
+  let(:fetcher) { double("fetcher") }
+  let(:manager) { double("manager") }
+  let(:thread) {double("thread")}
+  let(:consume_queue) { "TestQueue" }
+  let(:consumer) { TestConsumer }
+  let(:consumer_object) { consumer.new(consume_queue) }
+  let(:strategy) { Chore::Strategy::ThrottledConsumerStrategy.new(fetcher) }
+  let(:config) { double("config") }
+  let(:sized_queue) {double("sized_queue")}
+  let(:return_queue) {double("return_queue")}
+  let(:work) { double("work") }
+  let(:msg) { OpenStruct.new( :id => 1, :body => "test" ) }
+
+
+  before(:each) do
+    allow(fetcher).to receive(:consumers).and_return([consumer])
+    allow(fetcher).to receive(:manager).and_return(manager)
+    Chore.configure do |c| 
+      c.queues = [consume_queue] 
+      c.consumer = consumer
+    end
+  end
+
+  context '#fetch' do
+    it 'should call consume, \'@number_of_consumers\' number of times' do
+      allow(strategy).to receive(:consume).and_return(thread)
+      allow(thread).to receive(:join).and_return(true)
+      allow(Chore).to receive(:config).and_return(config)
+      allow(config).to receive(:queues).and_return([consume_queue])
+      strategy.instance_variable_set(:@consumers_per_queue, 5)
+      expect(strategy).to receive(:consume).with(consume_queue).exactly(5).times
+      strategy.fetch
+    end
+  end
+
+  context '#stop!' do
+    it 'should should stop itself, and every other consumer' do
+      allow(strategy).to receive(:running?).and_return(true)
+      strategy.instance_eval('@running = true')
+      strategy.instance_variable_set(:@consumers, [consumer_object])
+      expect(consumer_object).to receive(:stop)
+      strategy.stop!
+      expect(strategy.instance_variable_get(:@running)).to eq(false)
+    end
+  end
+
+  context '#provide_work' do
+    it 'should return upto n units of work' do
+      n = 2
+      strategy.instance_variable_set(:@queue, sized_queue)
+      allow(sized_queue).to receive(:size).and_return(10)
+      allow(sized_queue).to receive(:pop).and_return(work)
+      expect(sized_queue).to receive(:pop).exactly(n).times
+      res = strategy.provide_work(n)
+      expect(res.size).to eq(n)
+      expect(res).to be_a_kind_of(Array)
+    end
+
+    it 'should return an empty array if no work is found in the queue' do
+      n = 2
+      strategy.instance_variable_set(:@queue, sized_queue)
+      allow(sized_queue).to receive(:size).and_return(0)
+      allow(sized_queue).to receive(:pop).and_return(work)
+      expect(sized_queue).to receive(:pop).exactly(0).times
+      res = strategy.provide_work(n)
+      expect(res.size).to eq(0)
+      expect(res).to be_a_kind_of(Array)
+    end
+
+    it 'should return units of work from the return queue first' do
+      n = 2
+      strategy.instance_variable_set(:@return_queue, return_queue)
+      allow(return_queue).to receive(:empty?).and_return(false)
+      allow(return_queue).to receive(:size).and_return(10)
+      allow(return_queue).to receive(:pop).and_return(work)
+      expect(return_queue).to receive(:pop).exactly(n).times
+      res = strategy.provide_work(n)
+      expect(res.size).to eq(n)
+      expect(res).to be_a_kind_of(Array)
+    end
+
+    it 'should return units of work from all queues if return queue is small' do
+      n = 2
+
+      strategy.instance_variable_set(:@return_queue, return_queue)
+      allow(return_queue).to receive(:empty?).and_return(false, true)
+      allow(return_queue).to receive(:size).and_return(1)
+      allow(return_queue).to receive(:pop).and_return(work)
+      expect(return_queue).to receive(:pop).once
+
+      strategy.instance_variable_set(:@queue, sized_queue)
+      allow(sized_queue).to receive(:size).and_return(1)
+      allow(sized_queue).to receive(:pop).and_return(work)
+      expect(sized_queue).to receive(:pop).once
+
+      res = strategy.provide_work(n)
+      expect(res.size).to eq(n)
+      expect(res).to be_a_kind_of(Array)
+    end
+  end
+
+  context 'return_work' do
+    it 'should add it to the internal return queue' do
+      strategy.instance_variable_set(:@return_queue, [])
+      strategy.send(:return_work, [work])
+      strategy.send(:return_work, [work])
+      return_queue = strategy.instance_variable_get(:@return_queue)
+      expect(return_queue).to eq([work, work])
+    end
+  end
+
+  context '#consume' do
+    it 'should create a consumer object, add it to the list of consumers and start a consumer thread' do
+      allow(strategy).to receive(:start_consumer_thread).and_return(true)
+      expect(strategy).to receive(:start_consumer_thread)
+      strategy.send(:consume, consume_queue)
+      expect(strategy.instance_variable_get(:@consumers)).to be_a_kind_of(Array)
+      expect(strategy.instance_variable_get(:@consumers).first).to be_a_kind_of(TestConsumer)
+    end
+  end
+
+  context '#start_consumer_thread' do
+    let(:thread) { double('thread') }
+
+    it 'should create a thread' do
+      allow(Thread).to receive(:new).and_return(thread)
+      res = strategy.send(:start_consumer_thread, consumer_object)
+      expect(res).to eq(thread)
+    end
+  end
+
+  context '#create_work_units' do
+    it 'should create an unit of work from what the consumer gets, and adds it to the internal queue' do
+      strategy.instance_variable_set(:@queue, [])
+      res = strategy.send(:create_work_units, consumer_object)
+      internal_queue = strategy.instance_variable_get(:@queue)
+      expect(internal_queue).to be_a_kind_of(Array)
+      expect(internal_queue.first).to be_a_kind_of(Chore::UnitOfWork)
+      expect(internal_queue.first.id).to eq(msg)
+    end
+  end
+end

data/spec/chore/strategies/worker/forked_worker_strategy_spec.rb

@@ -8,11 +8,25 @@ describe Chore::Strategy::ForkedWorkerStrategy do
     allow(strategy).to receive(:exit!)
     strategy
   end
+  let(:consumer) { double('consumer', :complete => nil, :reject => nil) }
   let(:job_timeout) { 60 }
-  let(:job) { Chore::UnitOfWork.new(SecureRandom.uuid, 'test', job_timeout, Chore::Encoder::JsonEncoder.encode(TestJob.job_hash([1,2,"3"])), 0) }
+  let(:job) do
+    Chore::UnitOfWork.new(
+      SecureRandom.uuid,
+      'test',
+      job_timeout,
+      Chore::Encoder::JsonEncoder.encode(TestJob.job_hash([1,2,"3"])),
+      0,
+      consumer
+    )
+  end
   let!(:worker) { Chore::Worker.new(job) }
   let(:pid) { Random.rand(2048) }
 
+  before(:each) do
+    allow(consumer).to receive(:duplicate_message?).and_return(false)
+  end
+
   after(:each) do
     allow(Process).to receive(:kill) { nil }
     allow(Process).to receive(:wait) { pid }
@@ -164,6 +178,7 @@ describe Chore::Strategy::ForkedWorkerStrategy do
       end
 
       it 'should run the on_failure callback hook' do
+        allow(Chore).to receive(:run_hooks_for)
         forker.assign(job)
         expect(Chore).to receive(:run_hooks_for).with(:on_failure, anything, instance_of(Chore::TimeoutError))
         sleep 2

data/spec/chore/strategies/worker/helpers/ipc_spec.rb

@@ -0,0 +1,127 @@
+require 'spec_helper'
+require 'socket'
+
+describe Chore::Strategy::Ipc do
+  class DummyClass
+    include Chore::Strategy::Ipc
+  end
+
+  before(:each) do
+    @dummy_instance = DummyClass.new
+  end
+
+  let(:socket)          { double('socket') }
+  let(:connection)      { double('socket') }
+  let(:message)         { "test message" }
+  let(:encoded_message) { "#{[Marshal.dump(message).size].pack('L>')}#{Marshal.dump(message)}" }
+  let(:socket_base)     { './prefork_worker_sock-' }
+  let(:pid)             { '1' }
+  let(:socket_file)     { socket_base + pid }
+
+  describe '#create_master_socket' do
+    before(:each) do
+      allow(Process).to receive(:pid).and_return(pid)
+    end
+
+    it 'deletes socket file, if it already exists' do
+      allow(File).to receive(:exist?).with(socket_file).and_return(true)
+      allow(UNIXServer).to receive(:new).and_return(socket)
+      allow(socket).to receive(:setsockopt).and_return(true)
+      expect(File).to receive(:delete).with(socket_file)
+      @dummy_instance.create_master_socket
+    end
+
+    it 'should create and return a new UnixServer object' do
+      allow(File).to receive(:exist?).with(socket_file).and_return(false)
+      allow(UNIXServer).to receive(:new).and_return(socket)
+      allow(socket).to receive(:setsockopt).and_return(true)
+      expect(@dummy_instance.create_master_socket).to eq(socket)
+    end
+
+    it 'should set the required socket options' do
+      allow(File).to receive(:exist?).with(socket_file).and_return(false)
+      allow(UNIXServer).to receive(:new).and_return(socket)
+      expect(socket).to receive(:setsockopt).with(:SOCKET, :REUSEADDR, true)
+      @dummy_instance.create_master_socket
+    end
+  end
+
+  context '#child_connection' do
+    it 'should accept a connection and return the connection socket' do
+      expect(socket).to receive(:accept).and_return(connection)
+      expect(@dummy_instance.child_connection(socket)).to eq connection
+    end
+  end
+
+  context '#send_msg' do
+    it 'should raise an exception if the message is empty' do
+      expect { @dummy_instance.send_msg(socket, nil) }.to raise_error('send_msg cannot send empty messages')
+    end
+
+    it 'should send a message with the predefined protocol (size of message + marshalled message)' do
+      expect(socket).to receive(:send).with(encoded_message, 0)
+      @dummy_instance.send_msg(socket, message)
+    end
+  end
+
+  context '#read_msg' do
+    before(:each) do
+      allow(IO).to receive(:select).with([socket], nil, nil, 0.5).and_return([[socket], [], []])
+    end
+
+    it 'should return nil if the message size is missing' do
+      allow(socket).to receive(:recv).and_return(nil)
+      expect(@dummy_instance.read_msg(socket)).to eq(nil)
+    end
+
+    it 'should read a message with the predefined protocol (size of message + marshalled message)' do
+      allow(socket).to receive(:recv).and_return(encoded_message)
+      expect(@dummy_instance.read_msg(socket)).to eq(message)
+    end
+
+    it 'should raise an exception if the connection was dropped' do
+      allow(socket).to receive(:recv).and_raise(Errno::ECONNRESET)
+      expect { @dummy_instance.read_msg(socket) }.to raise_error(Errno::ECONNRESET)
+    end
+  end
+
+  context '#add_worker_socket' do
+    it 'should create and return a new UnixSocket object' do
+      allow(UNIXSocket).to receive(:new).and_return(socket)
+      allow(socket).to receive(:setsockopt).and_return(true)
+      expect(@dummy_instance.add_worker_socket).to eq(socket)
+    end
+
+    it 'should set the required socket options' do
+      allow(UNIXSocket).to receive(:new).and_return(socket)
+      expect(socket).to receive(:setsockopt).with(:SOCKET, :REUSEADDR, true)
+      @dummy_instance.add_worker_socket
+    end
+  end
+
+  context '#clear_ready' do
+    it 'should remove the ready signal from the the socket' do
+      expect(socket).to receive(:gets)
+      @dummy_instance.clear_ready(socket)
+    end
+  end
+
+  context '#signal_ready' do
+    it 'should set a ready signal on the socket' do
+      expect(socket).to receive(:puts).with('R')
+      @dummy_instance.signal_ready(socket)
+    end
+  end
+
+  context '#select_sockets' do
+    it 'should return a readable socket if one is found' do
+      allow(IO).to receive(:select).with([socket], nil, [socket], 0.5).and_return([[socket], [], []])
+      expect(@dummy_instance.select_sockets([socket], nil, 0.5)).to eq([[socket], [], []])
+    end
+
+    it 'should timeout and return no sockets if none are found within the timeout window' do
+      expect(@dummy_instance.select_sockets(nil, nil, 0.1)).to eq(nil)
+    end
+  end
+end
+

data/spec/chore/strategies/worker/helpers/preforked_worker_spec.rb

@@ -0,0 +1,236 @@
+require 'spec_helper'
+
+describe Chore::Strategy::PreforkedWorker do
+  before(:each) do
+    allow_any_instance_of(Chore::Strategy::PreforkedWorker).to receive(:post_fork_setup)
+  end
+
+  let(:preforkedworker) { Chore::Strategy::PreforkedWorker.new }
+  let(:socket)          { double("socket") }
+  let(:work)            { double("work") }
+  let(:consumer)        { double("consumer") }
+  let(:worker)          { double("worker") }
+  let(:config)          { double("config") }
+  let(:queues)          { double("queues") }
+  let(:consumer_object) { double("consumer_object") }
+
+  context '#start_worker' do
+    it 'should connect to the master to signal that it is ready, and process messages with the worker' do
+      allow(preforkedworker).to receive(:connect_to_master).and_return(socket)
+      expect(preforkedworker).to receive(:connect_to_master).with(socket)
+      allow(preforkedworker).to receive(:worker).and_return(true)
+      expect(preforkedworker).to receive(:worker).with(socket)
+      preforkedworker.start_worker(socket)
+    end
+  end
+
+  describe '#worker' do
+    before(:each) do
+      preforkedworker.instance_variable_set(:@self_read, socket)
+      allow(preforkedworker).to receive(:select_sockets).and_return([[socket],nil,nil])
+      allow(preforkedworker).to receive(:read_msg).and_return(nil)
+      allow(preforkedworker).to receive(:is_orphan?).and_return(false)
+      allow(preforkedworker).to receive(:process_work).and_return(true)
+      allow(preforkedworker).to receive(:signal_ready).and_return(true)
+      allow(work).to receive(:queue_timeout).and_return(20*60)
+    end
+
+
+    it 'should not run while @running is false' do
+      allow(preforkedworker).to receive(:running?).and_return(false)
+      expect(preforkedworker).to receive(:select_sockets).exactly(0).times
+      begin
+        preforkedworker.send(:worker, nil)
+      rescue SystemExit=>e
+        expect(e.status).to eq(0)
+      end
+    end
+
+    it 'should be able to handle timeouts on readable sockets' do    
+       allow(preforkedworker).to receive(:running?).and_return(true, true, false)   
+       allow(preforkedworker).to receive(:select_sockets).and_return(nil)   
+       expect(preforkedworker).to receive(:select_sockets).exactly(2).times   
+       begin    
+         preforkedworker.send(:worker, nil)   
+       rescue SystemExit=>e   
+         expect(e.status).to eq(0)    
+       end    
+     end    
+
+    it 'should read a message if the connection is readable' do
+      allow(preforkedworker).to receive(:running?).and_return(true, false)
+      expect(preforkedworker).to receive(:read_msg).once
+      begin
+        preforkedworker.send(:worker, nil)
+      rescue SystemExit=>e
+        expect(e.status).to eq(0)
+      end
+    end
+
+    it 'should check if the master is alive, and if not, it should end' do
+      allow(preforkedworker).to receive(:running).and_return(true)
+      allow(preforkedworker).to receive(:select_sockets).and_return([[socket],nil,nil])
+      allow(preforkedworker).to receive(:read_msg).and_return(nil)
+      allow(preforkedworker).to receive(:is_orphan?).and_return(true)
+      begin
+        preforkedworker.send(:worker, socket)
+      rescue SystemExit=>e
+        expect(e.status).to eq(0)
+      end
+    end
+
+    it 'should process work if it is read from the master and signal ready' do
+      allow(preforkedworker).to receive(:running?).and_return(true, false)
+      allow(preforkedworker).to receive(:read_msg).and_return(work)
+      expect(preforkedworker).to receive(:process_work).once
+      expect(preforkedworker).to receive(:signal_ready).once
+      begin
+        preforkedworker.send(:worker, socket)
+      rescue SystemExit=>e
+        expect(e.status).to eq(0)
+      end
+    end
+
+    it 'should exit the process if the connection to master is closed' do
+      allow(preforkedworker).to receive(:running?).and_return(true)
+      allow(preforkedworker).to receive(:read_msg).and_raise(Errno::ECONNRESET)
+      begin
+        preforkedworker.send(:worker, socket)
+      rescue SystemExit=>e
+        expect(e.status).to eq(0)
+      end
+    end
+  end
+
+  context '#connect_to_master' do
+    it 'should create a connection to the master, and send it its PID and a ready message' do
+      allow(preforkedworker).to receive(:child_connection).and_return(socket)
+      allow(preforkedworker).to receive(:send_msg).and_return(true)
+      allow(preforkedworker).to receive(:signal_ready).and_return(true)
+
+      expect(preforkedworker).to receive(:child_connection).once
+      expect(preforkedworker).to receive(:send_msg).once
+      expect(preforkedworker).to receive(:signal_ready).once
+
+      res = preforkedworker.send(:connect_to_master,socket)
+
+      expect(res).to eq(socket)
+    end
+  end
+
+  context '#post_fork_setup' do
+    before(:each) do
+      allow(preforkedworker).to receive(:procline).and_return(true)
+      allow(preforkedworker).to receive(:trap_signals)
+      allow_any_instance_of(Chore::Strategy::PreforkedWorker).to receive(:post_fork_setup).and_call_original
+    end
+
+    it 'should change the process name' do
+      expect(preforkedworker).to receive(:procline)
+      preforkedworker.send(:post_fork_setup)
+    end
+
+    it 'should trap new relevant signals' do
+      expect(preforkedworker).to receive(:trap_signals)
+      preforkedworker.send(:post_fork_setup)
+    end
+
+    it "should run after_fork hooks" do
+      hook_called = false
+      Chore.add_hook(:after_fork) { hook_called = true }
+      preforkedworker.send(:post_fork_setup)
+      expect(hook_called).to be true
+    end
+  end
+
+  context '#process_work' do
+    before(:each) do
+      allow(preforkedworker).to receive(:consumer).and_return(consumer)
+      allow(work).to receive(:queue_name).and_return("test_queue")
+      allow(work).to receive(:consumer=)
+      allow(work).to receive(:queue_timeout).and_return(10)
+      allow(Chore::Worker).to receive(:new).and_return(worker)
+      allow(worker).to receive(:start)
+    end
+
+    it 'should fetch the consumer object associated with the queue' do
+      expect(preforkedworker).to receive(:consumer)
+      preforkedworker.send(:process_work, [work])
+    end
+
+    it 'should create and start a worker object with the job sent to it' do
+      expect(worker).to receive(:start)
+      preforkedworker.send(:process_work, [work])
+    end
+
+    it 'should timeout if the work runs for longer than the queue timeout' do
+      allow(work).to receive(:queue_timeout).and_return(1)
+      allow(worker).to receive(:start) { sleep 5 }
+      begin
+        preforkedworker.send(:process_work, [work])
+      rescue => e
+        expect(e.class).to eq(Timeout::Error)
+      end
+    end
+  end
+
+  context '#consumer' do
+    before(:each) do
+      preforkedworker.instance_variable_set(:@consumer_cache, {key_1: :value_1})
+      allow(Chore).to receive(:config).and_return(config)
+      allow(config).to receive(:queues).and_return(queues)
+      allow(queues).to receive(:size).and_return(2)
+      allow(config).to receive(:consumer).and_return(consumer)
+      allow(consumer).to receive(:new).and_return(:value_2)
+    end
+
+    it 'should fetch a consumer object if it was created previously for this queue' do
+      preforkedworker.instance_variable_set(:@consumer_cache, {key_1: :value_1})
+      res = preforkedworker.send(:consumer,:key_1)
+      expect(res).to eq(:value_1)
+    end
+
+    it 'should create and return a new consumer object if one does not exist for this queue' do
+      preforkedworker.instance_variable_set(:@consumer_cache, {})
+      expect(consumer).to receive(:new)
+      res = preforkedworker.send(:consumer,:key_2)
+      expect(res).to eq(:value_2)
+    end
+  end
+
+  context '#trap_signals' do
+    it 'should reset signals' do
+      allow(Chore::Signal).to receive(:reset)
+      allow(Chore::Signal).to receive(:trap)
+      expect(Chore::Signal).to receive(:reset)
+      preforkedworker.send(:trap_signals)
+    end
+
+    it 'should trap the signals passed to it' do
+      allow(Chore::Signal).to receive(:reset)
+      allow(Chore::Signal).to receive(:trap)
+      expect(Chore::Signal).to receive(:trap).with(:INT).once
+      expect(Chore::Signal).to receive(:trap).with(:QUIT).once
+      expect(Chore::Signal).to receive(:trap).with(:TERM).once
+      expect(Chore::Signal).to receive(:trap).with(:USR1).once
+      preforkedworker.send(:trap_signals)
+    end
+  end
+
+  context '#is_orphan?' do
+    it 'should return true if the parent is dead' do
+      allow(Process).to receive(:ppid).and_return(10)
+      preforkedworker.instance_variable_set(:@manager_pid, 9)
+      res = preforkedworker.send(:is_orphan?)
+      expect(res).to eq(true)
+    end
+
+    it 'should return false if the parent is alive' do
+      allow(Process).to receive(:ppid).and_return(10)
+      preforkedworker.instance_variable_set(:@manager_pid, 10)
+      res = preforkedworker.send(:is_orphan?)
+      expect(res).to eq(false)
+    end
+  end
+end
+