chore-core 1.8.2 → 4.0.0

data/spec/chore/queues/sqs_spec.rb

@@ -1,53 +1,44 @@
 require 'spec_helper'
 
-module Chore
-  describe Queues::SQS do
-    context "when managing queues" do
-      let(:fake_sqs) {double(Object)}
-      let(:fake_queue_collection) {double(Object)}
-      let(:queue_name) {"test"}
-      let(:queue_url) {"http://amazon.sqs.url/queues/#{queue_name}"}
-      let(:fake_queue) {double(Object)}
-
-      before(:each) do
-        allow(AWS::SQS).to receive(:new).and_return(fake_sqs)
-        allow(Chore).to receive(:prefixed_queue_names).and_return([queue_name])
-        allow(fake_queue).to receive(:delete)
+describe Chore::Queues::SQS do
+  include_context 'fake objects'
+
+  context "when managing queues" do
+    before(:each) do
+      allow(Aws::SQS::Client).to receive(:new).and_return(sqs)
+      allow(sqs).to receive(:create_queue).and_return(queue)
+      allow(sqs).to receive(:delete_queue).and_return(Struct.new(nil))
+      allow(queue).to receive(:delete).and_return(sqs.delete_queue(queue))
+      allow(Chore).to receive(:prefixed_queue_names).and_return([queue_name])
+      allow(queue).to receive(:delete)
+    end
 
-        allow(fake_queue_collection).to receive(:[]).and_return(fake_queue)
-        allow(fake_queue_collection).to receive(:create)
-        allow(fake_queue_collection).to receive(:url_for).with(queue_name).and_return(queue_url)
+    it 'should create queues that are defined in its internal job name list' do
+      #Only one job defined in the spec suite
+      expect(sqs).to receive(:create_queue).with(queue_name: queue_name)
+      Chore::Queues::SQS.create_queues!
+    end
 
-        allow(fake_sqs).to receive(:queues).and_return(fake_queue_collection)
-      end
+    it 'should delete queues that are defined in its internal job name list' do
+      #Only one job defined in the spec suite
+      expect(sqs).to receive(:delete_queue).with(queue_url: sqs.get_queue_url.queue_url)
+      Chore::Queues::SQS.delete_queues!
+    end
 
-      it 'should create queues that are defined in its internal job name list' do
-        #Only one job defined in the spec suite
-        expect(fake_queue_collection).to receive(:create)
-        Chore::Queues::SQS.create_queues!
+    context 'and checking for existing queues' do
+      it 'checks for existing queues' do
+        expect(described_class).to receive(:existing_queues).and_return([])
+        Chore::Queues::SQS.create_queues!(true)
       end
 
-      it 'should delete queues that are defined in its internal job name list' do
-        #Only one job defined in the spec suite
-        expect(fake_queue).to receive(:delete)
-        Chore::Queues::SQS.delete_queues!
+      it 'raises an error if a queue does exist' do
+        allow(described_class).to receive(:existing_queues).and_return([queue_name])
+        expect{Chore::Queues::SQS.create_queues!(true)}.to raise_error(RuntimeError)
       end
 
-      context 'and checking for existing queues' do
-        it 'checks for existing queues' do
-          expect(described_class).to receive(:existing_queues).and_return([])
-          Chore::Queues::SQS.create_queues!(true)
-        end
-
-        it 'raises an error if a queue does exist' do
-          allow(described_class).to receive(:existing_queues).and_return([queue_name])
-          expect{Chore::Queues::SQS.create_queues!(true)}.to raise_error(RuntimeError)
-        end
-
-        it 'does not raise an error if a queue does not exist' do
-          allow(described_class).to receive(:existing_queues).and_return([])
-          expect{Chore::Queues::SQS.create_queues!(true)}.not_to raise_error
-        end
+      it 'does not raise an error if a queue does not exist' do
+        allow(described_class).to receive(:existing_queues).and_return([])
+        expect{Chore::Queues::SQS.create_queues!(true)}.not_to raise_error
       end
     end
   end

data/spec/chore/strategies/consumer/batcher_spec.rb

@@ -90,4 +90,54 @@ describe Chore::Strategy::Batcher do
       subject.batch.should == ['test']
     end
   end
+
+  describe 'schedule' do
+    let(:timeout) { 5 }
+    let(:batch) { [] }
+
+    before(:each) do
+      Thread.stub(:new) do |&block|
+        # Stop the batcher on the next iteration
+        subject.stub(:sleep) { subject.stop }
+
+        # Run the scheduling thread
+        block.call(timeout)
+      end
+
+      subject.batch = batch.dup
+    end
+
+    context 'with no items' do
+      it 'should not invoke the callback' do
+        callback.should_not_receive(:call)
+        subject.schedule(timeout)
+      end
+    end
+
+    context 'with new items' do
+      let(:batch) do
+        [
+          Chore::UnitOfWork.new.tap {|work| work.created_at = Time.now - 2}
+        ]
+      end
+
+      it 'should not invoke the callback' do
+        callback.should_not_receive(:call).with(batch)
+        subject.schedule(timeout)
+      end
+    end
+
+    context 'with old items' do
+      let(:batch) do
+        [
+          Chore::UnitOfWork.new.tap {|work| work.created_at = Time.now - 6}
+        ]
+      end
+
+      it 'should invoke the callback' do
+        callback.should_receive(:call).with(batch)
+        subject.schedule(timeout)
+      end
+    end
+  end
 end

data/spec/chore/strategies/consumer/single_consumer_strategy_spec.rb

@@ -11,13 +11,13 @@ describe Chore::Strategy::SingleConsumerStrategy do
     fetcher.stub(:manager) { manager }
     Chore.config.stub(:queues).and_return(test_queues)
     Chore.config.stub(:consumer).and_return(consumer)
-    
+
   end
 
   it "should consume and then assign a message" do
     consumer.should_receive(:new).with(test_queues.first).and_return(consumer)
-    consumer.should_receive(:consume).and_yield(1, 'test-queue', 60, "test", 0)
-    manager.should_receive(:assign).with(Chore::UnitOfWork.new(1, 'test-queue', 60, "test", 0, consumer))
+    consumer.should_receive(:consume).and_yield(1, nil, 'test-queue', 60, "test", 0)
+    manager.should_receive(:assign).with(Chore::UnitOfWork.new(1, nil, 'test-queue', 60, "test", 0, consumer))
     strategy.fetch
   end
 end

data/spec/chore/strategies/consumer/threaded_consumer_strategy_spec.rb

@@ -29,8 +29,8 @@ describe Chore::Strategy::ThreadedConsumerStrategy do
   before(:each) do
     fetcher.stub(:consumers) { [consumer] }
     fetcher.stub(:manager) { manager }
-    Chore.configure do |c| 
-      c.queues = ['test'] 
+    Chore.configure do |c|
+      c.queues = ['test']
       c.consumer = consumer
       c.batch_size = batch_size
     end
@@ -40,7 +40,7 @@ describe Chore::Strategy::ThreadedConsumerStrategy do
     let(:batch_size) { 2 }
 
     it "should queue but not assign the message" do
-      consumer.any_instance.should_receive(:consume).and_yield(1, 'test-queue', 60, "test", 0)
+      consumer.any_instance.should_receive(:consume).and_yield(1, nil, 'test-queue', 60, "test", 0)
       strategy.fetch
       strategy.batcher.batch.size.should == 1
 
@@ -51,6 +51,7 @@ describe Chore::Strategy::ThreadedConsumerStrategy do
       work.message.should == "test"
       work.previous_attempts.should == 0
       work.current_attempt.should == 1
+      work.created_at.should_not be_nil
     end
   end
 
@@ -59,7 +60,7 @@ describe Chore::Strategy::ThreadedConsumerStrategy do
 
     it "should assign the batch" do
       manager.should_receive(:assign)
-      consumer.any_instance.should_receive(:consume).and_yield(1, 'test-queue', 60, "test", 0)
+      consumer.any_instance.should_receive(:consume).and_yield(1, nil, 'test-queue', 60, "test", 0)
       strategy.fetch
       strategy.batcher.batch.size.should == 0
     end
@@ -89,8 +90,8 @@ describe Chore::Strategy::ThreadedConsumerStrategy do
 
     before do
       fetcher.stub(:consumers) { [bad_consumer] }
-      Chore.configure do |c| 
-        c.queues = ['test'] 
+      Chore.configure do |c|
+        c.queues = ['test']
         c.consumer = bad_consumer
         c.batch_size = batch_size
         c.threads_per_queue = 1

data/spec/chore/strategies/consumer/throttled_consumer_strategy_spec.rb

@@ -0,0 +1,165 @@
+require 'spec_helper'
+
+class TestConsumer < Chore::Consumer
+  def initialize(queue_name, opts={})
+  end
+
+  def consume
+    # just something that looks like an SQS message
+    msg = OpenStruct.new( :id => 1, :body => "test" )
+    yield msg if block_given?
+  end
+end
+
+class NoQueueConsumer < Chore::Consumer
+  def initialize(queue_name, opts={})
+    raise Chore::TerribleMistake
+  end
+
+  def consume
+  end
+end
+
+describe Chore::Strategy::ThrottledConsumerStrategy do
+  let(:fetcher) { double("fetcher") }
+  let(:manager) { double("manager") }
+  let(:thread) {double("thread")}
+  let(:consume_queue) { "TestQueue" }
+  let(:consumer) { TestConsumer }
+  let(:consumer_object) { consumer.new(consume_queue) }
+  let(:strategy) { Chore::Strategy::ThrottledConsumerStrategy.new(fetcher) }
+  let(:config) { double("config") }
+  let(:sized_queue) {double("sized_queue")}
+  let(:return_queue) {double("return_queue")}
+  let(:work) { double("work") }
+  let(:msg) { OpenStruct.new( :id => 1, :body => "test" ) }
+
+
+  before(:each) do
+    allow(fetcher).to receive(:consumers).and_return([consumer])
+    allow(fetcher).to receive(:manager).and_return(manager)
+    Chore.configure do |c| 
+      c.queues = [consume_queue] 
+      c.consumer = consumer
+    end
+  end
+
+  context '#fetch' do
+    it 'should call consume, \'@number_of_consumers\' number of times' do
+      allow(strategy).to receive(:consume).and_return(thread)
+      allow(thread).to receive(:join).and_return(true)
+      allow(Chore).to receive(:config).and_return(config)
+      allow(config).to receive(:queues).and_return([consume_queue])
+      strategy.instance_variable_set(:@consumers_per_queue, 5)
+      expect(strategy).to receive(:consume).with(consume_queue).exactly(5).times
+      strategy.fetch
+    end
+  end
+
+  context '#stop!' do
+    it 'should should stop itself, and every other consumer' do
+      allow(strategy).to receive(:running?).and_return(true)
+      strategy.instance_eval('@running = true')
+      strategy.instance_variable_set(:@consumers, [consumer_object])
+      expect(consumer_object).to receive(:stop)
+      strategy.stop!
+      expect(strategy.instance_variable_get(:@running)).to eq(false)
+    end
+  end
+
+  context '#provide_work' do
+    it 'should return upto n units of work' do
+      n = 2
+      strategy.instance_variable_set(:@queue, sized_queue)
+      allow(sized_queue).to receive(:size).and_return(10)
+      allow(sized_queue).to receive(:pop).and_return(work)
+      expect(sized_queue).to receive(:pop).exactly(n).times
+      res = strategy.provide_work(n)
+      expect(res.size).to eq(n)
+      expect(res).to be_a_kind_of(Array)
+    end
+
+    it 'should return an empty array if no work is found in the queue' do
+      n = 2
+      strategy.instance_variable_set(:@queue, sized_queue)
+      allow(sized_queue).to receive(:size).and_return(0)
+      allow(sized_queue).to receive(:pop).and_return(work)
+      expect(sized_queue).to receive(:pop).exactly(0).times
+      res = strategy.provide_work(n)
+      expect(res.size).to eq(0)
+      expect(res).to be_a_kind_of(Array)
+    end
+
+    it 'should return units of work from the return queue first' do
+      n = 2
+      strategy.instance_variable_set(:@return_queue, return_queue)
+      allow(return_queue).to receive(:empty?).and_return(false)
+      allow(return_queue).to receive(:size).and_return(10)
+      allow(return_queue).to receive(:pop).and_return(work)
+      expect(return_queue).to receive(:pop).exactly(n).times
+      res = strategy.provide_work(n)
+      expect(res.size).to eq(n)
+      expect(res).to be_a_kind_of(Array)
+    end
+
+    it 'should return units of work from all queues if return queue is small' do
+      n = 2
+
+      strategy.instance_variable_set(:@return_queue, return_queue)
+      allow(return_queue).to receive(:empty?).and_return(false, true)
+      allow(return_queue).to receive(:size).and_return(1)
+      allow(return_queue).to receive(:pop).and_return(work)
+      expect(return_queue).to receive(:pop).once
+
+      strategy.instance_variable_set(:@queue, sized_queue)
+      allow(sized_queue).to receive(:size).and_return(1)
+      allow(sized_queue).to receive(:pop).and_return(work)
+      expect(sized_queue).to receive(:pop).once
+
+      res = strategy.provide_work(n)
+      expect(res.size).to eq(n)
+      expect(res).to be_a_kind_of(Array)
+    end
+  end
+
+  context 'return_work' do
+    it 'should add it to the internal return queue' do
+      strategy.instance_variable_set(:@return_queue, [])
+      strategy.send(:return_work, [work])
+      strategy.send(:return_work, [work])
+      return_queue = strategy.instance_variable_get(:@return_queue)
+      expect(return_queue).to eq([work, work])
+    end
+  end
+
+  context '#consume' do
+    it 'should create a consumer object, add it to the list of consumers and start a consumer thread' do
+      allow(strategy).to receive(:start_consumer_thread).and_return(true)
+      expect(strategy).to receive(:start_consumer_thread)
+      strategy.send(:consume, consume_queue)
+      expect(strategy.instance_variable_get(:@consumers)).to be_a_kind_of(Array)
+      expect(strategy.instance_variable_get(:@consumers).first).to be_a_kind_of(TestConsumer)
+    end
+  end
+
+  context '#start_consumer_thread' do
+    let(:thread) { double('thread') }
+
+    it 'should create a thread' do
+      allow(Thread).to receive(:new).and_return(thread)
+      res = strategy.send(:start_consumer_thread, consumer_object)
+      expect(res).to eq(thread)
+    end
+  end
+
+  context '#create_work_units' do
+    it 'should create an unit of work from what the consumer gets, and adds it to the internal queue' do
+      strategy.instance_variable_set(:@queue, [])
+      res = strategy.send(:create_work_units, consumer_object)
+      internal_queue = strategy.instance_variable_get(:@queue)
+      expect(internal_queue).to be_a_kind_of(Array)
+      expect(internal_queue.first).to be_a_kind_of(Chore::UnitOfWork)
+      expect(internal_queue.first.id).to eq(msg)
+    end
+  end
+end

data/spec/chore/strategies/worker/forked_worker_strategy_spec.rb

@@ -8,11 +8,26 @@ describe Chore::Strategy::ForkedWorkerStrategy do
     allow(strategy).to receive(:exit!)
     strategy
   end
+  let(:consumer) { double('consumer', :complete => nil, :reject => nil) }
   let(:job_timeout) { 60 }
-  let(:job) { Chore::UnitOfWork.new(SecureRandom.uuid, 'test', job_timeout, Chore::Encoder::JsonEncoder.encode(TestJob.job_hash([1,2,"3"])), 0) }
+  let(:job) do
+    Chore::UnitOfWork.new(
+      SecureRandom.uuid,
+      nil,
+      'test',
+      job_timeout,
+      Chore::Encoder::JsonEncoder.encode(TestJob.job_hash([1,2,"3"])),
+      0,
+      consumer
+    )
+  end
   let!(:worker) { Chore::Worker.new(job) }
   let(:pid) { Random.rand(2048) }
 
+  before(:each) do
+    allow(consumer).to receive(:duplicate_message?).and_return(false)
+  end
+
   after(:each) do
     allow(Process).to receive(:kill) { nil }
     allow(Process).to receive(:wait) { pid }
@@ -164,6 +179,7 @@ describe Chore::Strategy::ForkedWorkerStrategy do
       end
 
       it 'should run the on_failure callback hook' do
+        allow(Chore).to receive(:run_hooks_for)
         forker.assign(job)
         expect(Chore).to receive(:run_hooks_for).with(:on_failure, anything, instance_of(Chore::TimeoutError))
         sleep 2
@@ -281,4 +297,3 @@ describe Chore::Strategy::ForkedWorkerStrategy do
     end
   end
 end
-

data/spec/chore/strategies/worker/helpers/ipc_spec.rb

@@ -0,0 +1,127 @@
+require 'spec_helper'
+require 'socket'
+
+describe Chore::Strategy::Ipc do
+  class DummyClass
+    include Chore::Strategy::Ipc
+  end
+
+  before(:each) do
+    @dummy_instance = DummyClass.new
+  end
+
+  let(:socket)          { double('socket') }
+  let(:connection)      { double('socket') }
+  let(:message)         { "test message" }
+  let(:encoded_message) { "#{[Marshal.dump(message).size].pack('L>')}#{Marshal.dump(message)}" }
+  let(:socket_base)     { './prefork_worker_sock-' }
+  let(:pid)             { '1' }
+  let(:socket_file)     { socket_base + pid }
+
+  describe '#create_master_socket' do
+    before(:each) do
+      allow(Process).to receive(:pid).and_return(pid)
+    end
+
+    it 'deletes socket file, if it already exists' do
+      allow(File).to receive(:exist?).with(socket_file).and_return(true)
+      allow(UNIXServer).to receive(:new).and_return(socket)
+      allow(socket).to receive(:setsockopt).and_return(true)
+      expect(File).to receive(:delete).with(socket_file)
+      @dummy_instance.create_master_socket
+    end
+
+    it 'should create and return a new UnixServer object' do
+      allow(File).to receive(:exist?).with(socket_file).and_return(false)
+      allow(UNIXServer).to receive(:new).and_return(socket)
+      allow(socket).to receive(:setsockopt).and_return(true)
+      expect(@dummy_instance.create_master_socket).to eq(socket)
+    end
+
+    it 'should set the required socket options' do
+      allow(File).to receive(:exist?).with(socket_file).and_return(false)
+      allow(UNIXServer).to receive(:new).and_return(socket)
+      expect(socket).to receive(:setsockopt).with(:SOCKET, :REUSEADDR, true)
+      @dummy_instance.create_master_socket
+    end
+  end
+
+  context '#child_connection' do
+    it 'should accept a connection and return the connection socket' do
+      expect(socket).to receive(:accept).and_return(connection)
+      expect(@dummy_instance.child_connection(socket)).to eq connection
+    end
+  end
+
+  context '#send_msg' do
+    it 'should raise an exception if the message is empty' do
+      expect { @dummy_instance.send_msg(socket, nil) }.to raise_error('send_msg cannot send empty messages')
+    end
+
+    it 'should send a message with the predefined protocol (size of message + marshalled message)' do
+      expect(socket).to receive(:send).with(encoded_message, 0)
+      @dummy_instance.send_msg(socket, message)
+    end
+  end
+
+  context '#read_msg' do
+    before(:each) do
+      allow(IO).to receive(:select).with([socket], nil, nil, 0.5).and_return([[socket], [], []])
+    end
+
+    it 'should return nil if the message size is missing' do
+      allow(socket).to receive(:recv).and_return(nil)
+      expect(@dummy_instance.read_msg(socket)).to eq(nil)
+    end
+
+    it 'should read a message with the predefined protocol (size of message + marshalled message)' do
+      allow(socket).to receive(:recv).and_return(encoded_message)
+      expect(@dummy_instance.read_msg(socket)).to eq(message)
+    end
+
+    it 'should raise an exception if the connection was dropped' do
+      allow(socket).to receive(:recv).and_raise(Errno::ECONNRESET)
+      expect { @dummy_instance.read_msg(socket) }.to raise_error(Errno::ECONNRESET)
+    end
+  end
+
+  context '#add_worker_socket' do
+    it 'should create and return a new UnixSocket object' do
+      allow(UNIXSocket).to receive(:new).and_return(socket)
+      allow(socket).to receive(:setsockopt).and_return(true)
+      expect(@dummy_instance.add_worker_socket).to eq(socket)
+    end
+
+    it 'should set the required socket options' do
+      allow(UNIXSocket).to receive(:new).and_return(socket)
+      expect(socket).to receive(:setsockopt).with(:SOCKET, :REUSEADDR, true)
+      @dummy_instance.add_worker_socket
+    end
+  end
+
+  context '#clear_ready' do
+    it 'should remove the ready signal from the the socket' do
+      expect(socket).to receive(:gets)
+      @dummy_instance.clear_ready(socket)
+    end
+  end
+
+  context '#signal_ready' do
+    it 'should set a ready signal on the socket' do
+      expect(socket).to receive(:puts).with('R')
+      @dummy_instance.signal_ready(socket)
+    end
+  end
+
+  context '#select_sockets' do
+    it 'should return a readable socket if one is found' do
+      allow(IO).to receive(:select).with([socket], nil, [socket], 0.5).and_return([[socket], [], []])
+      expect(@dummy_instance.select_sockets([socket], nil, 0.5)).to eq([[socket], [], []])
+    end
+
+    it 'should timeout and return no sockets if none are found within the timeout window' do
+      expect(@dummy_instance.select_sockets(nil, nil, 0.1)).to eq(nil)
+    end
+  end
+end
+