chore-core 1.10.0 → 4.0.0

data/spec/chore/queues/sqs/publisher_spec.rb

@@ -1,74 +1,63 @@
 require 'spec_helper'
 
-module Chore
-  describe Queues::SQS::Publisher do
-    let(:job) { {'class' => 'TestJob', 'args'=>[1,2,'3']}}
-    let(:queue_name) { 'test_queue' }
-    let(:queue_url) {"http://www.queue_url.com/test_queue"}
-    let(:queue) { double('queue', :send_message => nil) }
-    let(:sqs) do
-      double('sqs', :queues => double('queues', :named => queue, :url_for => queue_url, :[] => queue))
-    end
-    let(:publisher) { Queues::SQS::Publisher.new }
-    let(:pool) { double("pool") }
+describe Chore::Queues::SQS::Publisher do
+  include_context 'fake objects'
 
-    before(:each) do
-      AWS::SQS.stub(:new).and_return(sqs)
-    end
+  let(:publisher) { Chore::Queues::SQS::Publisher.new }
+  let(:job) { {'class' => 'TestJob', 'args'=>[1,2,'3']}}
+  let(:send_message_result) { double(Aws::SQS::Types::SendMessageResult, :data => job) }
 
-    it 'should configure sqs' do
-      Chore.config.stub(:aws_access_key).and_return('key')
-      Chore.config.stub(:aws_secret_key).and_return('secret')
+  before(:each) do
+    allow(Aws::SQS::Client).to receive(:new).and_return(sqs)
+    allow(sqs).to receive(:send_message).and_return(send_message_result)
+  end
 
-      AWS::SQS.should_receive(:new).with(
-        :access_key_id => 'key',
-        :secret_access_key => 'secret',
-        :logger => Chore.logger,
-        :log_level => :debug
-      )
-      publisher.publish(queue_name,job)
-    end
+  it 'should configure sqs' do
+    expect(Aws::SQS::Client).to receive(:new)
+    publisher.publish(queue_name,job)
+  end
 
-    it 'should create send an encoded message to the specified queue' do
-      queue.should_receive(:send_message).with(job.to_json)
-      publisher.publish(queue_name,job)
-    end
+  it 'should not create a new SQS client before every publish' do
+    expect(Aws::SQS::Client).to receive(:new).once
+    2.times { publisher.send(:queue, queue_name) }
+  end
 
-    it 'should lookup the queue when publishing' do
-      sqs.queues.should_receive(:url_for).with('test_queue')
-      publisher.publish('test_queue', job)
-    end
+  it 'should lookup the queue when publishing' do
+    expect(sqs).to receive(:get_queue_url).with(queue_name: queue_name)
+    publisher.publish(queue_name, job)
+  end
 
-    it 'should lookup multiple queues if specified' do
-      sqs.queues.should_receive(:url_for).with('test_queue')
-      sqs.queues.should_receive(:url_for).with('test_queue2')
-      publisher.publish('test_queue', job)
-      publisher.publish('test_queue2', job)
-    end
+  it 'should create send an encoded message to the specified queue' do
+    expect(sqs).to receive(:send_message).with(queue_url: queue_url, message_body: job.to_json)
+    publisher.publish(queue_name,job)
+  end
+
+  it 'should lookup multiple queues if specified' do
+    second_queue_name = queue_name + '2'
+    expect(sqs).to receive(:get_queue_url).with(queue_name: queue_name)
+    expect(sqs).to receive(:get_queue_url).with(queue_name: second_queue_name)
+
+    publisher.publish(queue_name, job)
+    publisher.publish(second_queue_name, job)
+  end
+
+  it 'should only lookup a named queue once' do
+    expect(sqs).to receive(:get_queue_url).with(queue_name: queue_name).once
+    4.times { publisher.publish(queue_name, job) }
+  end
 
-    it 'should only lookup a named queue once' do
-      sqs.queues.should_receive(:url_for).with('test_queue').once
-      2.times { publisher.publish('test_queue', job) }
+  describe '#reset_connection!' do
+    it 'should empty API client connection pool after a call to reset_connection!' do
+      expect(Aws).to receive(:empty_connection_pools!)
+      Chore::Queues::SQS::Publisher.reset_connection!
+      publisher.send(:queue, queue_name)
     end
 
-    describe '#reset_connection!' do
-      it 'should reset the connection after a call to reset_connection!' do
-        AWS::Core::Http::ConnectionPool.stub(:pools).and_return([pool])
-        pool.should_receive(:empty!)
-        Chore::Queues::SQS::Publisher.reset_connection!
-        publisher.queue(queue_name)
-      end
-  
-      it 'should not reset the connection between calls' do
-        sqs = publisher.queue(queue_name)
-        sqs.should be publisher.queue(queue_name)
-      end
-  
-      it 'should reconfigure sqs' do
-        Chore::Queues::SQS::Publisher.reset_connection!
-        AWS::SQS.should_receive(:new)
-        publisher.queue(queue_name)
-      end
+    # TODO: this test seems like basic identity (i.e. not even a real test)
+    it 'should not reset the connection between calls' do
+      expect(Aws).to receive(:empty_connection_pools!).once
+      Chore::Queues::SQS::Publisher.reset_connection!
+      4.times { publisher.send(:queue, queue_name ) }
     end
   end
 end

data/spec/chore/queues/sqs_spec.rb

@@ -1,53 +1,44 @@
 require 'spec_helper'
 
-module Chore
-  describe Queues::SQS do
-    context "when managing queues" do
-      let(:fake_sqs) {double(Object)}
-      let(:fake_queue_collection) {double(Object)}
-      let(:queue_name) {"test"}
-      let(:queue_url) {"http://amazon.sqs.url/queues/#{queue_name}"}
-      let(:fake_queue) {double(Object)}
-
-      before(:each) do
-        allow(AWS::SQS).to receive(:new).and_return(fake_sqs)
-        allow(Chore).to receive(:prefixed_queue_names).and_return([queue_name])
-        allow(fake_queue).to receive(:delete)
+describe Chore::Queues::SQS do
+  include_context 'fake objects'
+
+  context "when managing queues" do
+    before(:each) do
+      allow(Aws::SQS::Client).to receive(:new).and_return(sqs)
+      allow(sqs).to receive(:create_queue).and_return(queue)
+      allow(sqs).to receive(:delete_queue).and_return(Struct.new(nil))
+      allow(queue).to receive(:delete).and_return(sqs.delete_queue(queue))
+      allow(Chore).to receive(:prefixed_queue_names).and_return([queue_name])
+      allow(queue).to receive(:delete)
+    end
 
-        allow(fake_queue_collection).to receive(:[]).and_return(fake_queue)
-        allow(fake_queue_collection).to receive(:create)
-        allow(fake_queue_collection).to receive(:url_for).with(queue_name).and_return(queue_url)
+    it 'should create queues that are defined in its internal job name list' do
+      #Only one job defined in the spec suite
+      expect(sqs).to receive(:create_queue).with(queue_name: queue_name)
+      Chore::Queues::SQS.create_queues!
+    end
 
-        allow(fake_sqs).to receive(:queues).and_return(fake_queue_collection)
-      end
+    it 'should delete queues that are defined in its internal job name list' do
+      #Only one job defined in the spec suite
+      expect(sqs).to receive(:delete_queue).with(queue_url: sqs.get_queue_url.queue_url)
+      Chore::Queues::SQS.delete_queues!
+    end
 
-      it 'should create queues that are defined in its internal job name list' do
-        #Only one job defined in the spec suite
-        expect(fake_queue_collection).to receive(:create)
-        Chore::Queues::SQS.create_queues!
+    context 'and checking for existing queues' do
+      it 'checks for existing queues' do
+        expect(described_class).to receive(:existing_queues).and_return([])
+        Chore::Queues::SQS.create_queues!(true)
       end
 
-      it 'should delete queues that are defined in its internal job name list' do
-        #Only one job defined in the spec suite
-        expect(fake_queue).to receive(:delete)
-        Chore::Queues::SQS.delete_queues!
+      it 'raises an error if a queue does exist' do
+        allow(described_class).to receive(:existing_queues).and_return([queue_name])
+        expect{Chore::Queues::SQS.create_queues!(true)}.to raise_error(RuntimeError)
       end
 
-      context 'and checking for existing queues' do
-        it 'checks for existing queues' do
-          expect(described_class).to receive(:existing_queues).and_return([])
-          Chore::Queues::SQS.create_queues!(true)
-        end
-
-        it 'raises an error if a queue does exist' do
-          allow(described_class).to receive(:existing_queues).and_return([queue_name])
-          expect{Chore::Queues::SQS.create_queues!(true)}.to raise_error(RuntimeError)
-        end
-
-        it 'does not raise an error if a queue does not exist' do
-          allow(described_class).to receive(:existing_queues).and_return([])
-          expect{Chore::Queues::SQS.create_queues!(true)}.not_to raise_error
-        end
+      it 'does not raise an error if a queue does not exist' do
+        allow(described_class).to receive(:existing_queues).and_return([])
+        expect{Chore::Queues::SQS.create_queues!(true)}.not_to raise_error
       end
     end
   end

data/spec/chore/strategies/consumer/single_consumer_strategy_spec.rb

@@ -11,13 +11,13 @@ describe Chore::Strategy::SingleConsumerStrategy do
     fetcher.stub(:manager) { manager }
     Chore.config.stub(:queues).and_return(test_queues)
     Chore.config.stub(:consumer).and_return(consumer)
-    
+
   end
 
   it "should consume and then assign a message" do
     consumer.should_receive(:new).with(test_queues.first).and_return(consumer)
-    consumer.should_receive(:consume).and_yield(1, 'test-queue', 60, "test", 0)
-    manager.should_receive(:assign).with(Chore::UnitOfWork.new(1, 'test-queue', 60, "test", 0, consumer))
+    consumer.should_receive(:consume).and_yield(1, nil, 'test-queue', 60, "test", 0)
+    manager.should_receive(:assign).with(Chore::UnitOfWork.new(1, nil, 'test-queue', 60, "test", 0, consumer))
     strategy.fetch
   end
 end

data/spec/chore/strategies/consumer/threaded_consumer_strategy_spec.rb

@@ -29,8 +29,8 @@ describe Chore::Strategy::ThreadedConsumerStrategy do
   before(:each) do
     fetcher.stub(:consumers) { [consumer] }
     fetcher.stub(:manager) { manager }
-    Chore.configure do |c| 
-      c.queues = ['test'] 
+    Chore.configure do |c|
+      c.queues = ['test']
       c.consumer = consumer
       c.batch_size = batch_size
     end
@@ -40,7 +40,7 @@ describe Chore::Strategy::ThreadedConsumerStrategy do
     let(:batch_size) { 2 }
 
     it "should queue but not assign the message" do
-      consumer.any_instance.should_receive(:consume).and_yield(1, 'test-queue', 60, "test", 0)
+      consumer.any_instance.should_receive(:consume).and_yield(1, nil, 'test-queue', 60, "test", 0)
       strategy.fetch
       strategy.batcher.batch.size.should == 1
 
@@ -60,7 +60,7 @@ describe Chore::Strategy::ThreadedConsumerStrategy do
 
     it "should assign the batch" do
       manager.should_receive(:assign)
-      consumer.any_instance.should_receive(:consume).and_yield(1, 'test-queue', 60, "test", 0)
+      consumer.any_instance.should_receive(:consume).and_yield(1, nil, 'test-queue', 60, "test", 0)
       strategy.fetch
       strategy.batcher.batch.size.should == 0
     end
@@ -90,8 +90,8 @@ describe Chore::Strategy::ThreadedConsumerStrategy do
 
     before do
       fetcher.stub(:consumers) { [bad_consumer] }
-      Chore.configure do |c| 
-        c.queues = ['test'] 
+      Chore.configure do |c|
+        c.queues = ['test']
         c.consumer = bad_consumer
         c.batch_size = batch_size
         c.threads_per_queue = 1

data/spec/chore/strategies/consumer/throttled_consumer_strategy_spec.rb

@@ -0,0 +1,165 @@
+require 'spec_helper'
+
+class TestConsumer < Chore::Consumer
+  def initialize(queue_name, opts={})
+  end
+
+  def consume
+    # just something that looks like an SQS message
+    msg = OpenStruct.new( :id => 1, :body => "test" )
+    yield msg if block_given?
+  end
+end
+
+class NoQueueConsumer < Chore::Consumer
+  def initialize(queue_name, opts={})
+    raise Chore::TerribleMistake
+  end
+
+  def consume
+  end
+end
+
+describe Chore::Strategy::ThrottledConsumerStrategy do
+  let(:fetcher) { double("fetcher") }
+  let(:manager) { double("manager") }
+  let(:thread) {double("thread")}
+  let(:consume_queue) { "TestQueue" }
+  let(:consumer) { TestConsumer }
+  let(:consumer_object) { consumer.new(consume_queue) }
+  let(:strategy) { Chore::Strategy::ThrottledConsumerStrategy.new(fetcher) }
+  let(:config) { double("config") }
+  let(:sized_queue) {double("sized_queue")}
+  let(:return_queue) {double("return_queue")}
+  let(:work) { double("work") }
+  let(:msg) { OpenStruct.new( :id => 1, :body => "test" ) }
+
+
+  before(:each) do
+    allow(fetcher).to receive(:consumers).and_return([consumer])
+    allow(fetcher).to receive(:manager).and_return(manager)
+    Chore.configure do |c| 
+      c.queues = [consume_queue] 
+      c.consumer = consumer
+    end
+  end
+
+  context '#fetch' do
+    it 'should call consume, \'@number_of_consumers\' number of times' do
+      allow(strategy).to receive(:consume).and_return(thread)
+      allow(thread).to receive(:join).and_return(true)
+      allow(Chore).to receive(:config).and_return(config)
+      allow(config).to receive(:queues).and_return([consume_queue])
+      strategy.instance_variable_set(:@consumers_per_queue, 5)
+      expect(strategy).to receive(:consume).with(consume_queue).exactly(5).times
+      strategy.fetch
+    end
+  end
+
+  context '#stop!' do
+    it 'should should stop itself, and every other consumer' do
+      allow(strategy).to receive(:running?).and_return(true)
+      strategy.instance_eval('@running = true')
+      strategy.instance_variable_set(:@consumers, [consumer_object])
+      expect(consumer_object).to receive(:stop)
+      strategy.stop!
+      expect(strategy.instance_variable_get(:@running)).to eq(false)
+    end
+  end
+
+  context '#provide_work' do
+    it 'should return upto n units of work' do
+      n = 2
+      strategy.instance_variable_set(:@queue, sized_queue)
+      allow(sized_queue).to receive(:size).and_return(10)
+      allow(sized_queue).to receive(:pop).and_return(work)
+      expect(sized_queue).to receive(:pop).exactly(n).times
+      res = strategy.provide_work(n)
+      expect(res.size).to eq(n)
+      expect(res).to be_a_kind_of(Array)
+    end
+
+    it 'should return an empty array if no work is found in the queue' do
+      n = 2
+      strategy.instance_variable_set(:@queue, sized_queue)
+      allow(sized_queue).to receive(:size).and_return(0)
+      allow(sized_queue).to receive(:pop).and_return(work)
+      expect(sized_queue).to receive(:pop).exactly(0).times
+      res = strategy.provide_work(n)
+      expect(res.size).to eq(0)
+      expect(res).to be_a_kind_of(Array)
+    end
+
+    it 'should return units of work from the return queue first' do
+      n = 2
+      strategy.instance_variable_set(:@return_queue, return_queue)
+      allow(return_queue).to receive(:empty?).and_return(false)
+      allow(return_queue).to receive(:size).and_return(10)
+      allow(return_queue).to receive(:pop).and_return(work)
+      expect(return_queue).to receive(:pop).exactly(n).times
+      res = strategy.provide_work(n)
+      expect(res.size).to eq(n)
+      expect(res).to be_a_kind_of(Array)
+    end
+
+    it 'should return units of work from all queues if return queue is small' do
+      n = 2
+
+      strategy.instance_variable_set(:@return_queue, return_queue)
+      allow(return_queue).to receive(:empty?).and_return(false, true)
+      allow(return_queue).to receive(:size).and_return(1)
+      allow(return_queue).to receive(:pop).and_return(work)
+      expect(return_queue).to receive(:pop).once
+
+      strategy.instance_variable_set(:@queue, sized_queue)
+      allow(sized_queue).to receive(:size).and_return(1)
+      allow(sized_queue).to receive(:pop).and_return(work)
+      expect(sized_queue).to receive(:pop).once
+
+      res = strategy.provide_work(n)
+      expect(res.size).to eq(n)
+      expect(res).to be_a_kind_of(Array)
+    end
+  end
+
+  context 'return_work' do
+    it 'should add it to the internal return queue' do
+      strategy.instance_variable_set(:@return_queue, [])
+      strategy.send(:return_work, [work])
+      strategy.send(:return_work, [work])
+      return_queue = strategy.instance_variable_get(:@return_queue)
+      expect(return_queue).to eq([work, work])
+    end
+  end
+
+  context '#consume' do
+    it 'should create a consumer object, add it to the list of consumers and start a consumer thread' do
+      allow(strategy).to receive(:start_consumer_thread).and_return(true)
+      expect(strategy).to receive(:start_consumer_thread)
+      strategy.send(:consume, consume_queue)
+      expect(strategy.instance_variable_get(:@consumers)).to be_a_kind_of(Array)
+      expect(strategy.instance_variable_get(:@consumers).first).to be_a_kind_of(TestConsumer)
+    end
+  end
+
+  context '#start_consumer_thread' do
+    let(:thread) { double('thread') }
+
+    it 'should create a thread' do
+      allow(Thread).to receive(:new).and_return(thread)
+      res = strategy.send(:start_consumer_thread, consumer_object)
+      expect(res).to eq(thread)
+    end
+  end
+
+  context '#create_work_units' do
+    it 'should create an unit of work from what the consumer gets, and adds it to the internal queue' do
+      strategy.instance_variable_set(:@queue, [])
+      res = strategy.send(:create_work_units, consumer_object)
+      internal_queue = strategy.instance_variable_get(:@queue)
+      expect(internal_queue).to be_a_kind_of(Array)
+      expect(internal_queue.first).to be_a_kind_of(Chore::UnitOfWork)
+      expect(internal_queue.first.id).to eq(msg)
+    end
+  end
+end

data/spec/chore/strategies/worker/forked_worker_strategy_spec.rb

@@ -13,6 +13,7 @@ describe Chore::Strategy::ForkedWorkerStrategy do
   let(:job) do
     Chore::UnitOfWork.new(
       SecureRandom.uuid,
+      nil,
       'test',
       job_timeout,
       Chore::Encoder::JsonEncoder.encode(TestJob.job_hash([1,2,"3"])),
@@ -23,6 +24,10 @@ describe Chore::Strategy::ForkedWorkerStrategy do
   let!(:worker) { Chore::Worker.new(job) }
   let(:pid) { Random.rand(2048) }
 
+  before(:each) do
+    allow(consumer).to receive(:duplicate_message?).and_return(false)
+  end
+
   after(:each) do
     allow(Process).to receive(:kill) { nil }
     allow(Process).to receive(:wait) { pid }
@@ -174,6 +179,7 @@ describe Chore::Strategy::ForkedWorkerStrategy do
       end
 
       it 'should run the on_failure callback hook' do
+        allow(Chore).to receive(:run_hooks_for)
         forker.assign(job)
         expect(Chore).to receive(:run_hooks_for).with(:on_failure, anything, instance_of(Chore::TimeoutError))
         sleep 2
@@ -291,4 +297,3 @@ describe Chore::Strategy::ForkedWorkerStrategy do
     end
   end
 end
-