chore-core 1.5.2

data/spec/chore/strategies/consumer/threaded_consumer_strategy_spec.rb

@@ -0,0 +1,105 @@
+require 'spec_helper'
+
+class TestConsumer < Chore::Consumer
+  def initialize(queue_name, opts={})
+  end
+
+  def consume
+    # just something that looks like an SQS message
+    msg = OpenStruct.new( :id => 1, :body => "test" )
+    yield msg if block_given?
+  end
+end
+
+class NoQueueConsumer < Chore::Consumer
+  def initialize(queue_name, opts={})
+    raise Chore::TerribleMistake
+  end
+
+  def consume
+  end
+end
+
+describe Chore::Strategy::ThreadedConsumerStrategy do
+  let(:fetcher) { double("fetcher") }
+  let(:manager) { double("manager") }
+  let(:consumer) { TestConsumer }
+  let(:strategy) { Chore::Strategy::ThreadedConsumerStrategy.new(fetcher) }
+
+  before(:each) do
+    fetcher.stub(:consumers) { [consumer] }
+    fetcher.stub(:manager) { manager }
+    Chore.configure do |c| 
+      c.queues = ['test'] 
+      c.consumer = consumer
+      c.batch_size = batch_size
+    end
+  end
+
+  describe "unfilled batch" do
+    let(:batch_size) { 2 }
+
+    it "should queue but not assign the message" do
+      consumer.any_instance.should_receive(:consume).and_yield(1, 'test-queue', 60, "test", 0)
+      strategy.fetch
+      strategy.batcher.batch.size.should == 1
+
+      work = strategy.batcher.batch[0]
+      work.id.should == 1
+      work.queue_name.should == 'test-queue'
+      work.queue_timeout.should == 60
+      work.message.should == "test"
+      work.previous_attempts.should == 0
+      work.current_attempt.should == 1
+    end
+  end
+
+  describe "full batch" do
+    let(:batch_size) { 1 }
+
+    it "should assign the batch" do
+      manager.should_receive(:assign)
+      consumer.any_instance.should_receive(:consume).and_yield(1, 'test-queue', 60, "test", 0)
+      strategy.fetch
+      strategy.batcher.batch.size.should == 0
+    end
+  end
+
+  describe "2 threads per queue" do
+    let(:batch_size) { 2 }
+    let(:thread) { double('thread') }
+
+    before do
+      Chore.config.threads_per_queue = 2
+      thread.stub(:join)
+    end
+
+    it "should spawn two threads" do
+      # two for threads per queue and one for batcher#schedule
+      Thread.should_receive(:new).exactly(3).times { thread }
+      strategy.fetch
+    end
+  end
+
+  describe "non-existent queue" do
+    let(:bad_consumer) { NoQueueConsumer }
+    let(:fetcher) { double("fetcher") }
+    let(:strategy) { Chore::Strategy::ThreadedConsumerStrategy.new(fetcher) }
+    let(:batch_size) { 2 }
+
+    before do
+      fetcher.stub(:consumers) { [bad_consumer] }
+      Chore.configure do |c| 
+        c.queues = ['test'] 
+        c.consumer = bad_consumer
+        c.batch_size = batch_size
+        c.threads_per_queue = 1
+      end
+    end
+
+    it "should shut down when a queue doesn't exist" do
+      manager.should_receive(:shutdown!)
+      strategy.fetch
+    end
+  end
+end

data/spec/chore/strategies/worker/forked_worker_strategy_spec.rb

@@ -0,0 +1,281 @@
+require 'spec_helper'
+require 'securerandom'
+
+describe Chore::Strategy::ForkedWorkerStrategy do
+  let(:manager) { double('manager') }
+  let(:forker) do
+    strategy = Chore::Strategy::ForkedWorkerStrategy.new(manager)
+    strategy.stub(:exit!)
+    strategy
+  end
+  let(:job_timeout) { 60 }
+  let(:job) { Chore::UnitOfWork.new(SecureRandom.uuid, 'test', job_timeout, Chore::JsonEncoder.encode(TestJob.job_hash([1,2,"3"])), 0) }
+  let!(:worker) { Chore::Worker.new(job) }
+  let(:pid) { Random.rand(2048) }
+
+  after(:each) do
+    Process.stub(:kill => nil, :wait => pid)
+    forker.stop!
+  end
+
+  context "signal handling" do
+    it 'should trap signals from terminating children and reap them' do
+      Chore::Signal.should_receive(:trap).with('CHLD').and_yield
+      Chore::Strategy::ForkedWorkerStrategy.any_instance.should_receive(:reap_terminated_workers!)
+      forker
+    end
+  end
+
+  context '#assign' do
+    before(:each) do
+      forker.stub(:fork).and_yield.and_return(pid, pid + 1)
+      forker.stub(:after_fork)
+    end
+    after(:each) do
+      Chore.clear_hooks!
+    end
+
+    it 'should pop off the worker queue when assignd a job' do
+      Queue.any_instance.should_receive(:pop)
+      forker.assign(job)
+    end
+
+    it 'should assign a job to a new worker' do
+      Chore::Worker.should_receive(:new).with(job).and_return(worker)
+      worker.should_receive(:start)
+      forker.assign(job)
+    end
+
+    it 'should add an assigned worker to the worker list' do
+      forker.workers.should_receive(:[]=).with(pid,kind_of(Chore::Worker))
+      forker.assign(job)
+    end
+
+    it 'should fork a child for each new worker' do
+      forker.should_receive(:fork).and_yield.and_return(pid)
+      forker.assign(job)
+    end
+
+    it 'should remove the worker from the list when it has completed' do
+      forker.assign(job)
+
+      Process.should_receive(:wait).with(pid, Process::WNOHANG).and_return(pid)
+      forker.send(:reap_terminated_workers!)
+
+      forker.workers.should_not include(pid)
+    end
+
+    it 'should not remove the worker from the list if it has not yet completed' do
+      forker.assign(job)
+
+      Process.stub(:wait).and_return(nil)
+      forker.send(:reap_terminated_workers!)
+
+      forker.workers.should include(pid)
+    end
+
+    it 'should add the worker back to the queue when it has completed' do
+      2.times { forker.assign(job) }
+
+      Queue.any_instance.should_receive(:<<).twice.with(:worker)
+
+      Process.stub(:wait).and_return(pid, pid + 1)
+      forker.send(:reap_terminated_workers!)
+    end
+
+    it 'should only release a worker once if reaped twice' do
+      forker.assign(job)
+      reaped = false
+
+      forker.should_receive(:release_worker).once
+
+      Process.should_receive(:wait).twice.with(pid, anything).and_return do
+        if !reaped
+          reaped = true
+          forker.send(:reap_terminated_workers!)
+        end
+
+        pid
+      end
+      forker.send(:reap_terminated_workers!)
+    end
+
+    it 'should continue to allow reaping after an exception occurs' do
+      2.times { forker.assign(job) }
+
+      Process.should_receive(:wait).and_raise(Errno::ECHILD)
+      Process.should_receive(:wait).and_return(pid + 1)
+      forker.send(:reap_terminated_workers!)
+
+      forker.workers.should be_empty
+    end
+
+    [:before_fork, :after_fork, :within_fork, :before_fork_shutdown].each do |hook|
+      it "should run #{hook} hooks" do
+        hook_called = false
+        Chore.add_hook(hook) { hook_called = true }
+        forker.assign(job)
+        hook_called.should be_true
+      end
+    end
+
+    it 'should run around_fork hooks' do
+      hook_called = false
+      Chore.add_hook(:around_fork) {|&blk| hook_called = true; blk.call }
+      forker.assign(job)
+      hook_called.should be_true
+    end
+
+    it 'should run before_fork_shutdown hooks even if job errors' do
+      Chore::Worker.stub(:new).and_return(worker)
+      worker.stub(:start).and_raise(ArgumentError)
+
+      hook_called = false
+      Chore.add_hook(:before_fork_shutdown) { hook_called = true }
+
+      begin
+        forker.assign(job)
+      rescue ArgumentError => ex
+      end
+
+      hook_called.should be_true
+    end
+
+    it 'should exit the process without running at_exit handlers' do
+      forker.should_receive(:exit!).with(true)
+      forker.assign(job)
+    end
+
+    context 'long-lived work' do
+      let(:job_timeout) { 0.1 }
+
+      before(:each) do
+        Process.stub(:kill)
+        Chore::Worker.stub(:new).and_return(worker)
+      end
+
+      it 'should kill the process if it expires' do
+        Process.should_receive(:kill).with('KILL', pid)
+        forker.assign(job)
+        sleep 2
+      end
+
+      it 'should run the on_failure callback hook' do
+        forker.assign(job)
+        Chore.should_receive(:run_hooks_for).with(:on_failure, anything, instance_of(Chore::TimeoutError))
+        sleep 2
+      end
+    end
+
+    context 'short-lived work' do
+      let(:job_timeout) { 0.1 }
+
+      before(:each) do
+        Chore::Worker.stub(:new).and_return(worker)
+      end
+
+      it 'should not kill the process if does not expire' do
+        Process.should_not_receive(:kill)
+
+        forker.assign(job)
+        Process.stub(:wait).and_return(pid)
+        forker.send(:reap_terminated_workers!)
+        sleep 2
+      end
+    end
+  end
+
+  context '#before_fork' do
+    before(:each) do
+      Chore::Worker.stub(:new).and_return(worker)
+    end
+    after(:each) do
+      Chore.clear_hooks!
+    end
+
+    it 'should release the worker if an exception occurs' do
+      Chore.add_hook(:before_fork) { raise ArgumentError }
+      forker.should_receive(:release_worker)
+      forker.assign(job)
+    end
+  end
+
+  context '#around_fork' do
+    before(:each) do
+      Chore::Worker.stub(:new).and_return(worker)
+    end
+    after(:each) do
+      Chore.clear_hooks!
+    end
+
+    it 'should release the worker if an exception occurs' do
+      Chore.add_hook(:around_fork) {|worker, &block| raise ArgumentError}
+      forker.should_receive(:release_worker)
+      forker.assign(job)
+    end
+  end
+
+  context '#after_fork' do
+    let(:worker) { double('worker') }
+
+    it 'should clear signals' do
+      forker.should_receive(:clear_child_signals)
+      forker.send(:after_fork,worker)
+    end
+
+    it 'should trap signals' do
+      forker.should_receive(:trap_child_signals)
+      forker.send(:after_fork,worker)
+    end
+
+    it 'should set the procline' do
+      forker.should_receive(:procline)
+      forker.send(:after_fork,worker)
+    end
+  end
+
+  context '#stop!' do
+    before(:each) do
+      Process.stub(:kill)
+
+      forker.stub(:fork).and_yield.and_return(pid)
+      forker.stub(:after_fork)
+      forker.assign(job)
+    end
+
+    it 'should send a quit signal to each child' do
+      Process.should_receive(:kill).once.with('QUIT', pid)
+      Process.stub(:wait).and_return(pid, nil)
+      forker.stop!
+    end
+
+    it 'should reap each worker' do
+      Process.should_receive(:wait).and_return(pid)
+      forker.stop!
+      forker.workers.should be_empty
+    end
+
+    it 'should resend quit signal to children if workers are not reaped' do
+      Process.should_receive(:kill).twice.with('QUIT', pid)
+      Process.stub(:wait).and_return(nil, pid, nil)
+      forker.stop!
+    end
+
+    it 'should send kill signal to children if timeout is exceeded' do
+      Chore.config.stub(:shutdown_timeout).and_return(0.05)
+      Process.should_receive(:kill).once.with('QUIT', pid)
+      Process.stub(:wait).and_return(nil)
+      Process.should_receive(:kill).once.with('KILL', pid)
+      forker.stop!
+    end
+
+    it 'should not allow more work to be assigned' do
+      Process.stub(:wait).and_return(pid, nil)
+      forker.stop!
+
+      Chore::Worker.should_not_receive(:new)
+      forker.assign(job)
+    end
+  end
+end
+

data/spec/chore/strategies/worker/single_worker_strategy_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe Chore::Strategy::SingleWorkerStrategy do
+  let(:manager) { mock('Manager') }
+  subject       { described_class.new(manager) }
+
+  describe '#stop!' do
+    before(:each) do
+      subject.stub(:worker).and_return worker
+    end
+
+    context 'given there is no current worker' do
+      let(:worker)  { nil }
+
+      it 'does nothing' do
+        allow_message_expectations_on_nil
+        
+        worker.should_not_receive(:stop!)
+        subject.stop!
+      end
+    end
+
+    context 'given there is a current worker' do
+      let(:worker)  { mock('Worker') }
+      before(:each) do
+        subject.stub(:worker).and_return worker
+      end
+
+      it 'stops the worker' do
+        worker.should_receive(:stop!)
+        subject.stop!
+      end
+    end
+  end
+
+end

data/spec/chore/worker_spec.rb

@@ -0,0 +1,134 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe Chore::Worker do
+
+  class SimpleJob
+    include Chore::Job
+    queue_options :name => 'test', :publisher => FakePublisher, :max_attempts => 100
+
+    def perform(*args)
+      return args
+    end
+  end
+
+  let(:consumer) { double('consumer', :complete => nil) }
+  let(:job_args) { [1,2,'3'] }
+  let(:job) { SimpleJob.job_hash(job_args) }
+
+  it 'should use a default encoder' do
+    worker = Chore::Worker.new
+    worker.options[:encoder].should == Chore::JsonEncoder
+  end
+
+  it 'should process a single job' do
+    work = Chore::UnitOfWork.new('1', 'test', 60, Chore::JsonEncoder.encode(job), 0, consumer)
+    SimpleJob.should_receive(:perform).with(*job_args)
+    consumer.should_receive(:complete).with('1')
+    w = Chore::Worker.new(work)
+    w.start
+  end
+
+  it 'should process multiple jobs' do
+    work = []
+    10.times do |i|
+      work << Chore::UnitOfWork.new(i, 'test', 60, Chore::JsonEncoder.encode(job), 0, consumer)
+    end
+    SimpleJob.should_receive(:perform).exactly(10).times
+    consumer.should_receive(:complete).exactly(10).times
+    Chore::Worker.start(work)
+  end
+
+  describe 'expired?' do
+    let(:now) { Time.now }
+    let(:queue_timeouts) { [10, 20, 30] }
+    let(:work) do
+      queue_timeouts.map do |queue_timeout|
+        Chore::UnitOfWork.new('1', 'test', queue_timeout, Chore::JsonEncoder.encode(job), 0, consumer)
+      end
+    end
+    let(:worker) do
+      Timecop.freeze(now) do
+        Chore::Worker.new(work)
+      end
+    end
+
+    it 'should not be expired when before total timeout' do
+      worker.should_not be_expired
+    end
+
+    it 'should not be expired when at total timeout' do
+      Timecop.freeze(now + 60) do
+        worker.should_not be_expired
+      end
+    end
+
+    it 'should be expired when past total timeout' do
+      Timecop.freeze(now + 61) do
+        worker.should be_expired
+      end
+    end
+  end
+
+  describe 'with errors' do
+    context 'on parse' do
+      let(:job) { "Not-A-Valid-Json-String" }
+
+      it 'should fail cleanly' do
+        work = Chore::UnitOfWork.new(2,'test',60,job,0,consumer)
+        consumer.should_not_receive(:complete)
+        Chore.should_receive(:run_hooks_for).with(:on_failure, job, anything())
+        Chore::Worker.start(work)
+      end
+
+      context 'more than the maximum allowed times' do
+        before(:each) do
+          Chore.config.stub(:max_attempts).and_return(10)
+        end
+
+        it 'should permanently fail' do
+          work = Chore::UnitOfWork.new(2,'test',60,job,9,consumer)
+          Chore.should_receive(:run_hooks_for).with(:on_permanent_failure, 'test', job, anything())
+          Chore::Worker.start(work)
+        end
+
+        it 'should mark the item as completed' do
+          work = Chore::UnitOfWork.new(2,'test',60,job,9,consumer)
+          consumer.should_receive(:complete).with(2)
+          Chore::Worker.start(work)
+        end
+      end
+    end
+
+    context 'on perform' do
+      let(:encoded_job) { Chore::JsonEncoder.encode(job) }
+      let(:parsed_job) { JSON.parse(encoded_job) }
+
+      before(:each) do
+        SimpleJob.stub(:perform).and_raise(ArgumentError)
+        SimpleJob.stub(:run_hooks_for).and_return(true)
+      end
+
+      it 'should fail cleanly' do
+        work = Chore::UnitOfWork.new(2,'test',60,encoded_job,0,consumer)
+        consumer.should_not_receive(:complete)
+        SimpleJob.should_receive(:run_hooks_for).with(:on_failure, parsed_job, anything())
+
+        Chore::Worker.start(work)
+      end
+
+      context 'more than the maximum allowed times' do
+        it 'should permanently fail' do
+          work = Chore::UnitOfWork.new(2,'test',60,encoded_job,999,consumer)
+          SimpleJob.should_receive(:run_hooks_for).with(:on_permanent_failure, 'test', parsed_job, anything())
+          Chore::Worker.start(work)
+        end
+
+        it 'should mark the item as completed' do
+          work = Chore::UnitOfWork.new(2,'test',60,encoded_job,999,consumer)
+          consumer.should_receive(:complete).with(2)
+          Chore::Worker.start(work)
+        end
+      end
+    end
+  end
+end

data/spec/chore_spec.rb

@@ -0,0 +1,108 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe Chore do
+  before(:each) do
+    Chore.clear_hooks!
+  end
+  it 'should allow you to add a hook' do
+    blk = proc { true }
+    Chore.add_hook(:before_perform,&blk)
+    Chore.hooks_for(:before_perform).first.should == blk
+  end
+
+  it 'should call a hook if it exists' do
+    blk = proc { raise StandardError }
+    Chore.add_hook(:before_perform,&blk)
+    expect { Chore.run_hooks_for(:before_perform) }.to raise_error
+  end
+
+  it 'should not call a hook if it doesn\'t exist' do
+    blk = proc { raise StandardError }
+    expect { Chore.run_hooks_for(:before_perform) }.to_not raise_error
+  end
+
+  it 'should pass args to the block if present' do
+    blk = proc {|*args| true }
+    blk.should_receive(:call).with('1','2',3)
+    Chore.add_hook(:an_event,&blk)
+    Chore.run_hooks_for(:an_event, '1','2',3)
+  end
+
+  it 'should support multiple hooks for an event' do
+    blk = proc { true }
+    blk.should_receive(:call).twice
+    Chore.add_hook(:before_perform,&blk)
+    Chore.add_hook(:before_perform,&blk)
+
+    Chore.run_hooks_for(:before_perform)
+  end
+
+  it 'should support passing blocks' do
+    runner = proc { }
+
+    blk = proc { true }
+    blk.should_receive(:call) do |&arg1|
+      arg1.should_not be_nil
+    end
+    Chore.add_hook(:around_perform,&blk)
+
+    Chore.run_hooks_for(:around_perform, &runner)
+  end
+
+  it 'should call passed block' do
+    Chore.add_hook(:around_perform) do |&blk|
+      blk.call
+    end
+
+    run = false
+    Chore.run_hooks_for(:around_perform) { run = true }
+    run.should be_true
+  end
+
+  it 'should call passed blocks even if there are no hooks' do
+    run = false
+    Chore.run_hooks_for(:around_perform) { run = true }
+    run.should be_true
+  end
+
+  it 'should set configuration' do
+    Chore.configure {|c| c.test_config_option = 'howdy' }
+    Chore.config.test_config_option.should == 'howdy'
+  end
+
+  describe 'reopen_logs' do
+    let(:open_files) do
+      [
+        mock('file', :closed? => false, :reopen => nil, :sync= => nil, :path => '/a'),
+        mock('file2', :closed? => false, :reopen => nil, :sync= => nil, :path => '/b')
+      ]
+    end
+    let(:closed_files) do
+      [mock('file3', :closed? => true)]
+    end
+    let(:files) { open_files + closed_files }
+
+    before(:each) do
+      ObjectSpace.stub(:each_object).and_yield(open_files[0]).and_yield(open_files[1])
+    end
+
+    it 'should look up all instances of files' do
+      ObjectSpace.should_receive(:each_object).with(File)
+      Chore.reopen_logs
+    end
+
+    it 'should reopen files that are not closed' do
+      open_files.each do |file|
+        file.should_receive(:reopen).with(file.path, 'a+')
+      end
+      Chore.reopen_logs
+    end
+
+    it 'should sync files' do
+      open_files.each do |file|
+        file.should_receive(:sync=).with(true)
+      end
+      Chore.reopen_logs
+    end
+  end
+end