chore-core 1.5.2

data/spec/chore/cli_spec.rb

@@ -0,0 +1,182 @@
+require 'spec_helper'
+
+class TestJob2
+  include Chore::Job
+end
+
+describe Chore::CLI do
+  let(:cli) { Chore::CLI.send(:new) }
+
+  describe ".register_option" do
+    let(:cli) { Chore::CLI.instance }
+
+    it 'should allow configuration options to be registered externally' do
+      args = ['some','args']
+      Chore::CLI.register_option('option_name',*args)
+      cli.registered_opts['option_name'].should == {:args => args}
+    end
+
+    it 'should allow configuration options to come from a file' do
+      file = StringIO.new("--key-name=some_value")
+      File.stub(:read).and_return(file.read)
+
+      args = ['-k', '--key-name SOME_VALUE', "Some description"]
+      Chore::CLI.register_option "key_name", *args
+      options = cli.parse_config_file(file)
+      cli.registered_opts['key_name'].should == {:args => args}
+      options[:key_name].should == 'some_value'
+    end
+
+    it 'should handle ERB tags in a config file' do
+      file = StringIO.new("--key-name=<%= 'erb_inserted_value' %>\n--other-key=<%= 'second_val' %>")
+      File.stub(:read).and_return(file.read)
+
+      Chore::CLI.register_option "key_name", '-k', '--key-name SOME_VALUE', "Some description"
+      Chore::CLI.register_option "other_key", '-o', '--other-key SOME_VALUE', "Some description"
+      options = cli.parse_config_file(file)
+      options[:key_name].should == 'erb_inserted_value'
+      options[:other_key].should == 'second_val'
+    end
+  end
+
+  context 'queue mananagement' do
+    before(:each) do
+      TestJob.queue_options :name => 'test_queue', :publisher => Chore::Publisher
+      TestJob2.queue_options :name => 'test2', :publisher => Chore::Publisher
+      cli.send(:options).delete(:queues)
+      cli.stub(:validate!)
+      cli.stub(:boot_system)
+    end
+
+    after :all do
+      #Removing the prefix due to spec idempotency issues in job spec
+      Chore.config.queue_prefix = nil
+    end
+
+    it 'should detect queues based on included jobs' do
+      cli.parse([])
+      Chore.config.queues.should include('test_queue')
+    end
+
+    it 'should honor --except when processing all queues' do
+      cli.parse(['--except=test_queue'])
+      Chore.config.queues.should_not include('test_queue')
+    end
+
+    it 'should honor --queue-prefix when processing all queues' do
+      cli.parse(['--queue-prefix=prefixey'])
+      Chore.config.queues.should include('prefixey_test2')
+    end
+
+    context 'when provided duplicate queues' do
+      let(:queue_options) {['--queues=test2,test2']}
+      before :each do 
+        cli.parse(queue_options)
+      end
+
+      it 'should not have duplicate queues' do
+        Chore.config.queues.count.should == 1
+      end
+    end
+
+    context 'when both --queue_prefix and --queues have been provided' do
+      let(:queue_options) {['--queue-prefix=prefixy', '--queues=test2']}
+      before :each do
+        cli.parse(queue_options)
+      end
+
+      it 'should honor --queue_prefix' do
+        total_queues = Chore.config.queues.count
+        prefixed_queues = Chore.config.queues.count {|item| item.start_with?("prefixy_")}
+        prefixed_queues.should == total_queues
+      end
+
+      it 'should prefix the names of the specified queues' do
+        Chore.config.queues.should include('prefixy_test2')
+      end
+
+      it 'should not prefix the names of queues that were not specified' do
+        Chore.config.queues.should_not include('prefixy_test_queue')
+      end
+
+      it 'should not have a queue without the prefix' do
+        Chore.config.queues.should_not include('test2')
+      end
+
+      it 'should not have a queue that was not specified' do
+        Chore.config.queues.should_not include('test_queue')
+      end
+    end
+
+    it 'should raise an exception if both --queues and --except are specified' do
+      expect { cli.parse(['--except=something','--queues=something,else']) }.to raise_error(ArgumentError)
+    end
+
+    it 'should raise an exception if no queues are found' do
+      Chore::Job.job_classes.clear
+      expect { cli.parse([]) }.to raise_error(ArgumentError)
+    end
+  end
+
+  describe "#parse" do
+    let(:cli) do
+      Chore::CLI.send(:new).tap do |cli|
+        cli.send(:options).clear
+        cli.stub(:validate!)
+        cli.stub(:boot_system)
+        cli.stub(:detect_queues)
+      end
+    end
+
+    let(:config) { cli.parse(command); Chore.config }
+
+    context "--consumer-strategy" do
+      let(:command) { ["--consumer-strategy=Chore::Strategy::SingleConsumerStrategy"] }
+
+      it "should set the consumer class" do
+        config.consumer_strategy.should == Chore::Strategy::SingleConsumerStrategy
+      end
+    end
+
+    describe '--shutdown-timeout' do
+      let(:command) { ["--shutdown-timeout=#{amount}"] }
+      subject { config.shutdown_timeout }
+
+      context 'given a numeric value' do
+        let(:amount)  { '10.0' }
+
+        it 'is that amount' do
+          subject.should == amount.to_f
+        end
+      end
+
+      context 'given no value' do
+        let(:command) { [] }
+        it 'is the default value, 120 seconds' do
+          subject.should == 120.0
+        end
+      end
+    end
+
+    describe '--max-attempts' do
+      let(:command) { ["--max-attempts=#{amount}"] }
+      subject { config.max_attempts }
+
+      context 'given a numeric value' do
+        let(:amount)  { '10' }
+
+        it 'is that amount' do
+          subject.should == amount.to_i
+        end
+      end
+
+      context 'given no value' do
+        let(:command) { [] }
+        it 'is the default value, infinity' do
+          subject.should == 1.0 / 0.0
+        end
+      end
+    end
+  end
+
+end

data/spec/chore/consumer_spec.rb

@@ -0,0 +1,36 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe Chore::Consumer do
+  let(:queue) { "test" }
+  let(:options) { {} }
+  let(:consumer) { Chore::Consumer.new(queue) }
+  let(:message) { "message" }
+
+  it 'should have a consume method' do
+    consumer.should respond_to :consume
+  end
+
+  it 'should have a reject method' do
+    consumer.should respond_to :reject
+  end
+
+  it 'should have a complete method' do
+    consumer.should respond_to :complete
+  end
+
+  it 'should have a class level reset_connection method' do
+    Chore::Consumer.should respond_to :reset_connection!
+  end
+
+  it 'should not have an implemented consume method' do
+    expect { consumer.consume }.to raise_error(NotImplementedError)
+  end
+
+  it 'should not have an implemented reject method' do
+    expect { consumer.reject(message) }.to raise_error(NotImplementedError)
+  end
+
+  it 'should not have an implemented complete method' do
+    expect { consumer.complete(message) }.to raise_error(NotImplementedError)
+  end
+end

data/spec/chore/duplicate_detector_spec.rb

@@ -0,0 +1,62 @@
+require 'spec_helper'
+require 'securerandom'
+
+describe Chore::DuplicateDetector do
+  let(:memcache) { double("memcache") }
+  let(:dupe_on_cache_failure) { false }
+  let(:dedupe_params)  { { :memcached_client => memcache, :dupe_on_cache_failure => dupe_on_cache_failure } }
+  let(:dedupe) { Chore::DuplicateDetector.new(dedupe_params)}
+  let(:message) { double('message') }
+  let(:timeout) { 2 }
+  let(:queue_url) {"queue://bogus/url"}
+  let(:queue) { (q = double('queue')).stub(:visibility_timeout).and_return(timeout); q.stub(:url).and_return(queue_url); q }
+  let(:id) { SecureRandom.uuid }
+
+  before(:each) do
+    message.stub(:id).and_return(id)
+    message.stub(:queue).and_return(queue)
+  end
+
+  describe "#found_duplicate" do
+    it 'should not return true if the message has not already been seen' do
+      memcache.should_receive(:add).and_return(true)
+      dedupe.found_duplicate?(message).should_not be_true
+    end
+
+    it 'should return true if the message has already been seen' do
+      memcache.should_receive(:add).and_return(false)
+      dedupe.found_duplicate?(message).should be_true
+    end
+
+    it 'should return false if given an invalid message' do
+      dedupe.found_duplicate?(double()).should be_false
+    end
+
+    it "should return false when identity store errors" do
+      memcache.should_receive(:add).and_raise("no")
+      dedupe.found_duplicate?(message).should be_false
+    end
+
+    it "should set the timeout to be the queue's " do
+      memcache.should_receive(:add).with(id,"1",timeout).and_return(true)
+      dedupe.found_duplicate?(message).should be_false
+    end
+
+    it "should call #visibility_timeout once and only once" do
+      queue.should_receive(:visibility_timeout).once
+      memcache.should_receive(:add).at_least(3).times.and_return(true)
+      3.times { dedupe.found_duplicate?(message) }
+    end
+
+    context 'when a memecached connection error occurs' do
+      context 'and when Chore.config.dedupe_strategy is set to :strict' do
+        let(:dupe_on_cache_failure) { true }
+
+        it "returns true" do
+          memcache.should_receive(:add).and_raise
+          dedupe.found_duplicate?(message).should be_true
+        end
+      end
+    end
+  end
+end

data/spec/chore/fetcher_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+class TestConsumer < Chore::Consumer
+  def initialize(queue_name, opts={})
+  end
+
+  def consume
+    # just something that looks like an SQS message
+    msg = OpenStruct.new( :id => 1, :body => "test" )
+    yield msg if block_given?
+  end
+end
+
+describe Chore::Fetcher do
+  let(:manager) { double("manager") }
+  let(:consumer) { TestConsumer }
+  let(:fetcher) { Chore::Fetcher.new(manager) }
+
+  before(:each) do
+    Chore.configure do |c| 
+      c.queues = ['test'] 
+      c.consumer = consumer
+      c.batch_size = 1
+    end
+  end
+
+
+  it "should have a start function" do
+    fetcher.should respond_to :start
+  end
+
+  describe "fetching messages" do
+    it "should assign its message" do
+      manager.should_receive(:assign)
+      fetcher.start
+    end
+  end
+end

data/spec/chore/hooks_spec.rb

@@ -0,0 +1,44 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+class TestHooks
+  include Chore::Hooks
+end
+
+describe Chore::Hooks do
+  let(:test_instance){ TestHooks.new }
+  it 'should respond_to run_hooks_for' do
+    test_instance.should respond_to(:run_hooks_for)
+  end
+
+  it 'should call a defined hook' do
+    test_instance.should_receive(:before_perform_test).and_return(true)
+    test_instance.run_hooks_for(:before_perform)
+  end
+
+  it 'should call multiple defined hooks' do
+    3.times do |i|
+      test_instance.should_receive(:"before_perform_test#{i}").and_return(true)
+    end
+    test_instance.run_hooks_for(:before_perform)
+  end
+
+  it 'should bubble up raised exceptions' do
+    test_instance.should_receive(:"before_perform_raise").and_raise(RuntimeError)
+    expect { test_instance.run_hooks_for(:before_perform) }.to raise_error(RuntimeError)
+  end
+
+  describe 'global hooks' do
+    before(:each) do
+      Chore.clear_hooks!
+      Chore.add_hook(:test_hook,&callback)
+    end
+
+    let(:callback) { proc { true } }
+
+    it 'should call a global hook' do
+      callback.should_receive(:call).once
+      test_instance.run_hooks_for(:test_hook)
+    end
+
+  end
+end

data/spec/chore/job_spec.rb

@@ -0,0 +1,80 @@
+require 'spec_helper'
+
+describe Chore::Job do
+  let(:args) { [1,2, { :a => :hash }] }
+  let(:config) { { :name => 'test_queue', :publisher => Chore::Publisher } }
+
+  before(:each) do
+    TestJob.queue_options config
+  end
+
+  after(:each) do
+    TestJob.queue_options config
+  end
+
+  it 'should have an perform_async method' do
+    TestJob.should respond_to :perform_async
+  end
+
+  it 'should have a perform method' do
+    TestJob.should respond_to :perform
+  end
+
+  it 'should require a queue when configuring' do
+    expect { TestJob.queue_options(:name => nil) }.to raise_error(ArgumentError)
+  end
+
+  it 'should require a publisher when configuring' do
+    expect { TestJob.queue_options(:publisher => nil) }.to raise_error(ArgumentError)
+  end
+
+  it 'should take params via perform' do
+    TestJob.any_instance.should_receive(:perform).with(*args)
+    TestJob.perform(*args)
+  end
+
+  it 'should store class level configuration' do
+    TestJob.queue_options(:name => 'test_queue')
+    TestJob.options[:name].should == 'test_queue'
+  end
+
+  describe(:perform_async) do 
+    it 'should call an instance of the queue_options publisher' do
+      args = [1,2,{:h => 'ash'}]
+      TestJob.queue_options(:publisher => Chore::Publisher)
+      Chore::Publisher.any_instance.should_receive(:publish).with('test_queue',{:class => 'TestJob',:args => args}).and_return(true)
+      TestJob.perform_async(*args)
+    end
+  end
+
+  describe 'publisher configured via Chore.configure' do
+    before do
+      Chore.configure do |c|
+        c.publisher = Chore::Publisher
+      end
+
+      class NoPublisherJob
+        include Chore::Job
+        queue_options :name => "test_queue"
+
+        def perform
+        end
+      end
+    end
+
+    it 'should have the default publisher' do
+      NoPublisherJob.options[:publisher].should == Chore::Publisher
+    end
+
+    describe 'global publisher can be overridden' do
+      before do
+        TestJob.queue_options config.merge(:publisher => FakePublisher)
+      end
+
+      it 'should override publisher' do
+        TestJob.options[:publisher].should == FakePublisher
+        TestJob.options[:publisher].should_not == Chore::Publisher
+      end
+    end
+  end
+end

data/spec/chore/json_encoder_spec.rb

@@ -0,0 +1,11 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe Chore::JsonEncoder do
+  it 'should have an encode method' do
+    subject.should respond_to :encode
+  end
+
+  it 'should have a decode method' do
+    subject.should respond_to :decode
+  end
+end

data/spec/chore/manager_spec.rb

@@ -0,0 +1,39 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'timeout'
+
+describe Chore::Manager do
+
+  let(:fetcher) { mock(:start => nil) }
+  let(:opts) { { :num_workers => 4, :other_opt => 'hi', :fetcher => fetcher } }
+
+  before(:each) do
+    Chore.configure {|c| c.fetcher = fetcher; c.worker_strategy = Chore::Strategy::SingleWorkerStrategy }
+    fetcher.should_receive(:new).and_return(fetcher)
+  end
+
+  it 'should call create an instance of the defined fetcher' do
+    manager = Chore::Manager.new
+  end
+
+  describe 'running the manager' do
+
+    let(:manager) { Chore::Manager.new}
+    let(:work) { Chore::UnitOfWork.new(Chore::JsonEncoder.encode({:class => 'MyClass',:args => []}),mock()) }
+
+    it 'should start the fetcher when starting the manager' do
+      fetcher.should_receive(:start)
+      manager.start
+    end
+
+    describe 'assigning messages' do
+      it 'should create a worker if one is available' do
+        worker = mock()
+        Chore::Worker.should_receive(:new).with(work).and_return(worker)
+        worker.should_receive(:start).with()
+        manager.assign(work)
+      end
+    end
+
+  end
+
+end

data/spec/chore/queues/filesystem/filesystem_consumer_spec.rb

@@ -0,0 +1,71 @@
+require 'spec_helper'
+
+# This test is actually testing both the publisher and the consumer behavior but what we
+# really want to validate is that they can pass messages off to each other. Hard coding in
+# the behavior of each in two separate tests was becoming a mess and would be hard to maintain.
+describe Chore::Queues::Filesystem::Consumer do
+  let(:consumer) { Chore::Queues::Filesystem::Consumer.new(test_queue) }
+  let(:publisher) { Chore::Queues::Filesystem::Publisher.new }
+  let(:test_queues_dir) { "test-queues" }
+  let(:test_queue) { "test-queue" }
+
+  before do
+    Chore.config.fs_queue_root = test_queues_dir
+    Chore.config.stub(:default_queue_timeout).and_return(60)
+    consumer.stub(:sleep)
+  end
+  
+  after do
+    FileUtils.rm_rf(test_queues_dir)
+  end
+  
+  let!(:consumer_run_for_one_message) { consumer.stub(:running?).and_return(true, false) }
+  let(:test_job_hash) {{:class => "TestClass", :args => "test-args"}}
+
+  context "founding a published job" do
+    before do
+      publisher.publish(test_queue, test_job_hash)
+    end
+
+    it "should consume a published job and yield the job to the handler block" do
+      expect { |b| consumer.consume(&b) }.to yield_with_args(anything, 'test-queue', 60, test_job_hash.to_json, 0)
+    end
+
+    context "rejecting a job" do
+      let!(:consumer_run_for_two_messages) { consumer.stub(:running?).and_return(true, false,true,false) }
+    
+      it "should requeue a job that gets rejected" do
+        rejected = false
+        consumer.consume do |job_id, queue_name, job_hash|
+          consumer.reject(job_id)
+          rejected = true
+        end
+        rejected.should be_true
+
+        expect { |b| consumer.consume(&b) }.to yield_with_args(anything, 'test-queue', 60, test_job_hash.to_json, 1)
+      end
+    end
+    
+    context "completing a job" do
+      let!(:consumer_run_for_two_messages) { consumer.stub(:running?).and_return(true, false,true,false) }
+    
+      it "should remove job on completion" do
+        completed = false
+        consumer.consume do |job_id, queue_name, job_hash|
+          consumer.complete(job_id)
+          completed = true
+        end
+        completed.should be_true
+
+        expect { |b| consumer.consume(&b) }.to_not yield_control
+      end
+    end
+  end
+
+  context "not finding a published job" do
+    it "should consume a published job and yield the job to the handler block" do
+      expect { |b| consumer.consume(&b) }.to_not yield_control
+    end
+  end
+end
+