chore-core 1.5.2 → 1.5.10

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    ZDg1NjQ5Yzk0YzUwOGI5NDYxNWYyYmVhNTU4MmY5MTk2NWFjZjZmNA==
-  data.tar.gz: !binary |-
-    YmU5ZjI5YTFlNWUyODE3NzZlYzlkNjQ5NTFlZDViOWI4ZTZiYzU3Yg==
+SHA1:
+  metadata.gz: b1aae723bbfe580eb5d31876ca91f32dd28baae7
+  data.tar.gz: 014aae64a27e2a6eb6210b76561e6cc1765056b6
 SHA512:
-  metadata.gz: !binary |-
-    OWRlNjA2ZTI2Mjg2ZGRiYjk5MzY4Nzk1ZjNlMTMzM2UyMzU5NTdhZDI0ZDI2
-    Nzg1NTRkNGViZWI4NjgxNzkwYWQ4ZDM1YWU4MmQyMTgwNzRhMmYwMDRlYmMx
-    OTM3MDY5NWY5YjllZmZiM2I2NDJjZGE4NWRlZmY3MDcyYTI3Njg=
-  data.tar.gz: !binary |-
-    ODk0ZjIxYTI2YTU1MWQ0M2RiYzIyOGE4MGJjNTJkOTI1MmNhYjk4ZDFhODZl
-    YTQ5ZDIwMjAzOTEyY2JjYzBjNTZlMmFhYmEyZjYwYTU3Y2ZiMWUwYjZiYmRh
-    ZDg1YTg4MjBlZTkyNjQyMmRkNTZmZDkwMDBmYjFiNWJmYThiYjE=
+  metadata.gz: 110b80593989c4ddd8bf1502705d1ce88fd26ba0e458dcffdd9072ddcbf2d3dd9b7b4760b3a7ab37bbd150f66b4acea02e280e7396c2e203330820e5dceb6939
+  data.tar.gz: ef5522be2d88a900151437d3a127b5a692a7e71a3a77169a3fe12e25089157b363f7a0f890d1f48d4fd84aca305eaacfbb5f1553026a02a92ce99ef159ccf4c9

data/README.md

@@ -29,6 +29,7 @@ Other options include:
     --worker-strategy Chore::Strategy::ForkedWorkerStrategy # which worker strategy class to use
     --consumer Chore::Queues::SQS::Consumer # which consumer class to use Options are SQS::Consumer and Filesystem::Consumer. Filesystem is recommended for local and testing purposes only.
     --consumer-strategy Chore::Queues::Strategies::Consumer::ThreadedConsumerStrategy # which consuming strategy to use. Options are SingleConsumerStrategy and ThreadedConsumerStrategy. Threaded is recommended for better tuning your consuming profile
+    --consumer-sleep-interval 1.0 # The amount of time in seconds to sleep when a consumer doesn't receive any messages. Sub-second values are accepted. The default varies by consumer implementation. This is a weak form of backoff for when there is no work to do.
     --threads-per-queue 4 # number of threads per queue for consuming from a given queue.
     --dedupe-servers # if using SQS or similiar queue with at-least once delivery and your memcache is running on something other than localhost
     --batch-size 50 # how many messages are batched together before handing them to a worker

data/chore-core.gemspec

@@ -37,7 +37,7 @@ Gem::Specification.new do |s|
   s.summary = "Job processing... for the future!"
 
   s.add_runtime_dependency(%q<json>, [">= 0"])
-  s.add_runtime_dependency(%q<aws-sdk>, ["~> 1.12", ">= 1.12.0"])
+  s.add_runtime_dependency(%q<aws-sdk>, ["~> 1.56", ">= 1.56.0"])
   s.add_runtime_dependency(%q<thread>, ["~> 0.1.3"])
   s.add_development_dependency(%q<rspec>, ["~> 2.12.0"])
   s.add_development_dependency(%q<rdoc>, ["~> 3.12"])

data/lib/chore/cli.rb

@@ -92,7 +92,8 @@ module Chore #:nodoc:
     private
     def setup_options #:nodoc:
       register_option "queues", "-q", "--queues QUEUE1,QUEUE2", "Names of queues to process (default: all known)" do |arg|
-        options[:queues] = arg.split(",")
+        # This will remove duplicates. We ultimately force this to be a Set further below
+        options[:queues] = Set.new(arg.split(","))
       end
 
       register_option "except_queues", "-x", "--except QUEUE1,QUEUE2", "Process all queues (cannot specify --queues), except for the ones listed here" do |arg|
@@ -130,6 +131,12 @@ module Chore #:nodoc:
         options[:consumer_strategy] = constantize(arg)
       end
 
+      register_option 'consumer_sleep_interval', '--consumer-sleep-interval INTERVAL', Float, 'Length of time in seconds to sleep when the consumer does not find any messages. Defaults vary depending on consumer implementation'
+
+      register_option 'payload_handler', '--payload_handler CLASS_NAME', 'Name of a class to use as the payload handler (default: Chore::Job)' do |arg|
+        options[:payload_handler] = constantize(arg)
+      end
+
       register_option 'shutdown_timeout', '--shutdown-timeout SECONDS', Float, "Upon shutdown, the number of seconds to wait before force killing worker strategies (default: #{Chore::DEFAULT_OPTIONS[:shutdown_timeout]})"
 
       register_option 'dupe_on_cache_failure', '--dupe-on-cache-failure BOOLEAN', 'Determines the deduping behavior when a cache connection error occurs. When set to false, the message is assumed not to be a duplicate. (default: false)'
@@ -187,23 +194,36 @@ module Chore #:nodoc:
         raise ArgumentError, "Cannot specify both --except and --queues"
       end
 
-      if !options[:queues]
-        options[:queues] = Set.new
-        Chore::Job.job_classes.each do |j|
-          klazz = constantize(j)
-          options[:queues] << klazz.options[:name] if klazz.options[:name]
-          options[:queues] -= (options[:except_queues] || [])
-        end
+      ### Ensure after loading the app, that we have the prefix set (quirk of using Chore::Job#prefixed_queue_name to do the prefixing)
+      Chore.config.queue_prefix ||= options[:queue_prefix]
+
+      ### For ease, make sure except_queues is an array
+      options[:except_queues] ||= []
+
+      ### Generate a hash of all possible queues and their prefixed_names
+      queue_map = Chore::Job.job_classes.inject({}) do |hsh,j|
+        klazz = constantize(j)
+        hsh[klazz.options[:name]] = klazz.prefixed_queue_name if klazz.options[:name]
+        hsh
       end
 
-      original_queues = options[:queues].dup
-      # Now apply the prefixing
-      # Because the prefix could have been detected via the apps chore config file
-      # Lets see if that is present before we check for a CLI passed prefix
-      prefix = Chore.config.queue_prefix || options[:queue_prefix]
-      options[:queues] = Set.new.tap do |queue_set|
-        original_queues.each {|oq_name| queue_set << "#{prefix}#{oq_name}"}
+      ### If we passed in a queues option, use it as our working set, otherwise use all the queues
+      if options[:queues]
+        queues_to_use = options[:queues]
+      else
+        queues_to_use = queue_map.keys
       end
+
+      ### Remove the excepted queues from our working set
+      queues_to_use = queues_to_use - options[:except_queues]
+
+      ### Set options[:queues] to the prefixed value of the current working set
+      options[:queues] = queues_to_use.inject([]) do |queues,k|
+        raise ArgumentError, "You have specified a queue #{k} for which you have no corresponding Job class" unless queue_map.has_key?(k)
+        queues << queue_map[k]
+        queues
+      end
+
       raise ArgumentError, "No queues specified. Either include classes that include Chore::Job, or specify the --queues option" if options[:queues].empty?
     end
 

data/lib/chore/duplicate_detector.rb

@@ -29,11 +29,11 @@ module Chore
     # Checks the message against the configured dedupe server to see if the message is unique or not
     # Unique messages will return false
     # Duplicated messages will return true
-    def found_duplicate?(msg)
-      return false unless msg && msg.respond_to?(:queue) && msg.queue
-      timeout = self.queue_timeout(msg.queue)
+    def found_duplicate?(msg_data)
+      return false unless msg_data && msg_data[:queue]
+      timeout = self.queue_timeout(msg_data)
       begin
-        !@memcached_client.add(msg.id, "1",timeout)
+        !@memcached_client.add(msg_data[:id], "1",timeout)
       rescue StandardError => e
         if @dupe_on_cache_failure
           Chore.logger.error "Error accessing duplicate cache server. Assuming message is a duplicate. #{e}\n#{e.backtrace * "\n"}"
@@ -48,8 +48,8 @@ module Chore
     # Retrieves the timeout for the given queue. The timeout is the window of time in seconds that
     # we would consider the message to be non-unique, before we consider it dead in the water
     # After that timeout, we would consider the next copy of the message received to be unique, and process it.
-    def queue_timeout(queue)
-      @timeouts[queue.url] ||= queue.visibility_timeout || @timeout
+    def queue_timeout(msg_data)
+      @timeouts[msg_data[:queue]] ||= msg_data[:visibility_timeout] || @timeout
     end
 
   end

data/lib/chore/encoders/json_encoder.rb

@@ -0,0 +1,20 @@
+require 'json'
+
+module Chore
+  module Encoder
+    # Json encoding for serializing jobs. 
+    module JsonEncoder
+      class << self
+        # Encodes the +job+ into JSON using the standard ruby JSON parsing library
+        def encode(job)
+          JSON.generate(job.to_hash)
+        end
+
+        # Decodes the +job+ from JSON into a ruby Hash using the standard ruby JSON parsing library
+        def decode(job)
+          JSON.parse(job)
+        end
+      end
+    end
+  end
+end

data/lib/chore/job.rb

@@ -1,10 +1,13 @@
 require 'chore/hooks'
+require 'chore/util'
+require 'chore/encoders/json_encoder'
 
 module Chore
 
   # <tt>Chore::Job</tt> is the module which gives your job classes the methods they need to be published
   # and run within Chore. You cannot have a Job in Chore that does not include this module
   module Job
+    extend Util
 
     # An exception to represent a job choosing to forcibly reject a given instance of itself.
     # The reasoning behind rejecting the job and the message that spawned it are left to
@@ -25,6 +28,18 @@ module Chore
       base.extend(Hooks)
     end
 
+    def self.payload_class(message)
+      constantize(message['class'])
+    end
+
+    def self.decode(data)
+      Encoder::JsonEncoder.decode(data)
+    end
+
+    def self.payload(message)
+      message['args']
+    end
+
     module ClassMethods
       DEFAULT_OPTIONS = { }
       

data/lib/chore/publisher.rb

@@ -1,7 +1,7 @@
 module Chore
   # Base class for Chore Publishers. Provides the bare interface one needs to adhere to when writing custom publishers
   class Publisher
-    DEFAULT_OPTIONS = { :encoder => JsonEncoder }
+    DEFAULT_OPTIONS = { :encoder => Encoder::JsonEncoder }
 
     attr_accessor :options
 

data/lib/chore/queues/sqs/consumer.rb

@@ -34,10 +34,10 @@ module Chore
           while running?
             begin
               messages = handle_messages(&handler)
-              sleep 1 if messages.empty?
+              sleep (Chore.config.consumer_sleep_interval || 1) if messages.empty?
             rescue AWS::SQS::Errors::NonExistentQueue => e
-              Chore.logger.error "You specified a queue that does not exist. You must create the queue before starting Chore. Shutting down..."
-              raise Chore::TerribleMistake  
+              Chore.logger.error "You specified a queue '#{queue_name}' that does not exist. You must create the queue before starting Chore. Shutting down..."
+              raise Chore::TerribleMistake
             rescue => e
               Chore.logger.error { "SQSConsumer#Consume: #{e.inspect} #{e.backtrace * "\n"}" }
             end
@@ -61,18 +61,17 @@ module Chore
         # hook will be invoked, per message
         def handle_messages(&block)
           msg = queue.receive_messages(:limit => sqs_polling_amount, :attributes => [:receive_count])
-
           messages = *msg
           messages.each do |message|
-            block.call(message.handle, queue_name, queue_timeout, message.body, message.receive_count - 1) unless duplicate_message?(message)
+            block.call(message.handle, queue_name, queue_timeout, message.body, message.receive_count - 1) unless duplicate_message?({:id=>message.id, :queue=>message.queue.url, :visibility_timeout=>message.queue.visibility_timeout})
             Chore.run_hooks_for(:on_fetch, message.handle, message.body)
           end
           messages
         end
 
         # Checks if the given message has already been received within the timeout window for this queue
-        def duplicate_message?(message)
-          dupe_detector.found_duplicate?(message)
+        def duplicate_message?(msg_data)
+          dupe_detector.found_duplicate?(msg_data)
         end
 
         # Returns the instance of the DuplicateDetector used to ensure unique messages.

data/lib/chore/queues/sqs.rb

@@ -27,9 +27,14 @@ module Chore
         #This will raise an exception if AWS has not been configured by the project making use of Chore
         sqs_queues = AWS::SQS.new.queues
         Chore.prefixed_queue_names.each do |queue_name|
-          Chore.logger.info "Chore Deleting Queue: #{queue_name}"
-          url = sqs_queues.url_for(queue_name)
-          sqs_queues[url].delete
+          begin
+            Chore.logger.info "Chore Deleting Queue: #{queue_name}"
+            url = sqs_queues.url_for(queue_name)
+            sqs_queues[url].delete
+          rescue => e
+            # This could fail for a few reasons - log out why it failed, then continue on
+            Chore.logger.error "Deleting Queue: #{queue_name} failed because #{e}"
+          end
         end
         Chore.prefixed_queue_names
       end

data/lib/chore/strategies/worker/forked_worker_strategy.rb

@@ -5,7 +5,8 @@ module Chore
     class ForkedWorkerStrategy #:nodoc:
       attr_accessor :workers
 
-      def initialize(manager)
+      def initialize(manager, opts={})
+        @options = opts
         @manager = manager
         @stopped = false
         @workers = {}
@@ -57,7 +58,7 @@ module Chore
         return unless acquire_worker
 
         begin
-          w = Worker.new(work)
+          w = Worker.new(work, @options)
           Chore.run_hooks_for(:before_fork,w)
           pid = nil
           Chore.run_hooks_for(:around_fork,w) do

data/lib/chore/strategies/worker/single_worker_strategy.rb

@@ -7,7 +7,8 @@ module Chore
 
       attr_reader :worker
 
-      def initialize(manager)
+      def initialize(manager, opts={})
+        @options = opts
         @manager = manager
         @worker = nil
       end
@@ -23,7 +24,7 @@ module Chore
       # Assigns work if there isn't already a worker in progress. Otherwise, is a noop
       def assign(work)
         if workers_available?
-          @worker = Worker.new(work)
+          @worker = Worker.new(work, @options)
           @worker.start
           @worker = nil
           true

data/lib/chore/version.rb

@@ -2,7 +2,7 @@ module Chore
   module Version #:nodoc:
     MAJOR = 1
     MINOR = 5
-    PATCH = 2
+    PATCH = 10
 
     STRING = [ MAJOR, MINOR, PATCH ].join('.')
   end

data/lib/chore/worker.rb

@@ -1,5 +1,5 @@
 require 'chore/util'
-require 'chore/json_encoder'
+require 'chore/job'
 
 module Chore
   class TimeoutError < StandardError
@@ -12,24 +12,23 @@ module Chore
   class Worker
     include Util
 
-    DEFAULT_OPTIONS = { :encoder => JsonEncoder }
     attr_accessor :options
     attr_reader   :work
     attr_reader   :started_at
 
-    def self.start(work) #:nodoc:
-      self.new(work).start
+    def self.start(work, opts={}) #:nodoc:
+      self.new(work, opts).start
     end
 
     # Create a Worker. Give it an array of work (or single item), and +opts+.
-    # Currently, the only option supported by Worker is +:encoder+ which should match
-    # the +:encoder+ used by the Publisher who created the message.
+    # Currently, the only option supported by Worker is +:payload_handler+ which contains helpers
+    # for decoding the item and finding the correct payload class 
     def initialize(work=[],opts={})
       @stopping = false
       @started_at = Time.now
       @work = work
       @work = [work] unless work.kind_of?(Array)
-      self.options = DEFAULT_OPTIONS.merge(opts)
+      self.options = {:payload_handler => Chore.config.payload_handler}.merge(opts)
     end
 
     # Whether this worker has existed for longer than it's allowed to
@@ -56,7 +55,6 @@ module Chore
     def start
       @work.each do |item|
         return if @stopping
-        Chore.logger.debug { "Doing: #{item.queue_name} with #{item.message}" }
         begin
           start_item(item)
         rescue => e
@@ -76,20 +74,16 @@ module Chore
       @stopping = true
     end
 
-  protected
-    def payload_class(message)
-      constantize(message['class'])
-    end
-
   private
     def start_item(item)
-      message = decode_job(item.message)
-      klass = payload_class(message)
+      message = options[:payload_handler].decode(item.message)
+      klass = options[:payload_handler].payload_class(message)
       return unless klass.run_hooks_for(:before_perform,message)
-
       begin
+        Chore.logger.info { "Running job #{klass} with params #{message}"}
         perform_job(klass,message)
         item.consumer.complete(item.id)
+        Chore.logger.info { "Finished job #{klass} with params #{message}"}
         klass.run_hooks_for(:after_perform,message)
       rescue Job::RejectMessageException
         item.consumer.reject(item.id)
@@ -107,11 +101,7 @@ module Chore
     end
 
     def perform_job(klass, message)
-      klass.perform(*message['args'])
-    end
-
-    def decode_job(data)
-      options[:encoder].decode(data)
+      klass.perform(*options[:payload_handler].payload(message))
     end
   end
 end

data/lib/chore.rb

@@ -8,7 +8,7 @@ require 'chore/configuration'
 require 'chore/cli'
 require 'chore/consumer'
 require 'chore/job'
-require 'chore/json_encoder'
+require 'chore/encoders/json_encoder'
 require 'chore/manager'
 require 'chore/publisher'
 require 'chore/util'
@@ -21,6 +21,7 @@ require 'chore/publisher'
 end
 
 module Chore #:nodoc:
+  extend Util
   VERSION = Chore::Version::STRING #:nodoc:
 
   # The default configuration options for Chore.
@@ -38,7 +39,8 @@ module Chore #:nodoc:
     :default_queue_timeout => (12 * 60 * 60), # 12 hours
     :shutdown_timeout      => (2 * 60),
     :max_attempts          => 1.0 / 0.0, # Infinity
-    :dupe_on_cache_failure => false
+    :dupe_on_cache_failure => false,
+    :payload_handler => Chore::Job
   }
 
   class << self
@@ -211,7 +213,7 @@ module Chore #:nodoc:
 
   # List of queue_names as configured via Chore::Job including their prefix, if set.
   def self.prefixed_queue_names
-    Chore::Job.job_classes.collect {|klass| c = klass.constantize; c.prefixed_queue_name}
+    Chore::Job.job_classes.collect {|klass| c = constantize(klass); c.prefixed_queue_name}
   end
 end
 

data/spec/chore/cli_spec.rb

@@ -79,6 +79,14 @@ describe Chore::CLI do
       end
     end
 
+    context 'when provided a queue for which there is no job class' do
+      let(:queue_options) {['--queues=test2,test3']}
+
+      it 'should raise an error' do
+        expect {cli.parse(queue_options)}.to raise_error
+      end
+    end
+
     context 'when both --queue_prefix and --queues have been provided' do
       let(:queue_options) {['--queue-prefix=prefixy', '--queues=test2']}
       before :each do
@@ -112,9 +120,14 @@ describe Chore::CLI do
       expect { cli.parse(['--except=something','--queues=something,else']) }.to raise_error(ArgumentError)
     end
 
-    it 'should raise an exception if no queues are found' do
-      Chore::Job.job_classes.clear
-      expect { cli.parse([]) }.to raise_error(ArgumentError)
+    context "when no queues are found" do
+      before :each do
+        Chore::Job.stub(:job_classes).and_return([])
+      end
+
+      it 'should raise an exception' do
+        expect { cli.parse([]) }.to raise_error(ArgumentError)
+      end
     end
   end
 
@@ -138,6 +151,14 @@ describe Chore::CLI do
       end
     end
 
+    context "--payload_handler" do
+      let(:command) {["--payload_handler=Chore::Job"]}
+
+      it "should set the payload handler class" do
+        config.payload_handler.should == Chore::Job
+      end
+    end
+
     describe '--shutdown-timeout' do
       let(:command) { ["--shutdown-timeout=#{amount}"] }
       subject { config.shutdown_timeout }
@@ -158,6 +179,34 @@ describe Chore::CLI do
       end
     end
 
+    describe '--consumer_sleep_interval' do
+      let(:command) {["--consumer-sleep-interval=#{amount}"]}
+      subject {config.consumer_sleep_interval}
+
+      context 'given an integer value' do
+        let(:amount)  { '10' }
+
+        it 'is that amount' do
+          subject.should == amount.to_i
+        end
+      end
+
+      context 'given a float value' do
+        let(:amount)  { '0.5' }
+
+        it 'is that amount' do
+          subject.should == amount.to_f
+        end
+      end
+
+      context 'given no value' do
+        let(:command) { [] }
+        it 'is the default value, nil' do
+          subject.should == nil
+        end
+      end
+    end
+
     describe '--max-attempts' do
       let(:command) { ["--max-attempts=#{amount}"] }
       subject { config.max_attempts }

data/spec/chore/duplicate_detector_spec.rb

@@ -6,46 +6,42 @@ describe Chore::DuplicateDetector do
   let(:dupe_on_cache_failure) { false }
   let(:dedupe_params)  { { :memcached_client => memcache, :dupe_on_cache_failure => dupe_on_cache_failure } }
   let(:dedupe) { Chore::DuplicateDetector.new(dedupe_params)}
-  let(:message) { double('message') }
   let(:timeout) { 2 }
   let(:queue_url) {"queue://bogus/url"}
-  let(:queue) { (q = double('queue')).stub(:visibility_timeout).and_return(timeout); q.stub(:url).and_return(queue_url); q }
+  let(:queue) { double('queue', :visibility_timeout=>timeout, :url=>queue_url) }
   let(:id) { SecureRandom.uuid }
-
-  before(:each) do
-    message.stub(:id).and_return(id)
-    message.stub(:queue).and_return(queue)
-  end
+  let(:message) { double('message', :id=>id, :queue=>queue) }
+  let(:message_data) {{:id=>message.id, :visibility_timeout=>queue.visibility_timeout, :queue=>queue.url}}
 
   describe "#found_duplicate" do
     it 'should not return true if the message has not already been seen' do
       memcache.should_receive(:add).and_return(true)
-      dedupe.found_duplicate?(message).should_not be_true
+      dedupe.found_duplicate?(message_data).should_not be_true
     end
 
     it 'should return true if the message has already been seen' do
       memcache.should_receive(:add).and_return(false)
-      dedupe.found_duplicate?(message).should be_true
+      dedupe.found_duplicate?(message_data).should be_true
     end
 
     it 'should return false if given an invalid message' do
-      dedupe.found_duplicate?(double()).should be_false
+      dedupe.found_duplicate?({}).should be_false
     end
 
     it "should return false when identity store errors" do
       memcache.should_receive(:add).and_raise("no")
-      dedupe.found_duplicate?(message).should be_false
+      dedupe.found_duplicate?(message_data).should be_false
     end
 
     it "should set the timeout to be the queue's " do
       memcache.should_receive(:add).with(id,"1",timeout).and_return(true)
-      dedupe.found_duplicate?(message).should be_false
+      dedupe.found_duplicate?(message_data).should be_false
     end
 
     it "should call #visibility_timeout once and only once" do
       queue.should_receive(:visibility_timeout).once
       memcache.should_receive(:add).at_least(3).times.and_return(true)
-      3.times { dedupe.found_duplicate?(message) }
+      3.times { dedupe.found_duplicate?(message_data) }
     end
 
     context 'when a memecached connection error occurs' do
@@ -54,7 +50,7 @@ describe Chore::DuplicateDetector do
 
         it "returns true" do
           memcache.should_receive(:add).and_raise
-          dedupe.found_duplicate?(message).should be_true
+          dedupe.found_duplicate?(message_data).should be_true
         end
       end
     end

data/spec/chore/{json_encoder_spec.rb → encoders/json_encoder_spec.rb}

@@ -1,6 +1,6 @@
-require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
 
-describe Chore::JsonEncoder do
+describe Chore::Encoder::JsonEncoder do
   it 'should have an encode method' do
     subject.should respond_to :encode
   end

data/spec/chore/manager_spec.rb

@@ -15,10 +15,12 @@ describe Chore::Manager do
     manager = Chore::Manager.new
   end
 
+  
+
   describe 'running the manager' do
 
     let(:manager) { Chore::Manager.new}
-    let(:work) { Chore::UnitOfWork.new(Chore::JsonEncoder.encode({:class => 'MyClass',:args => []}),mock()) }
+    let(:work) { Chore::UnitOfWork.new(Chore::Encoder::JsonEncoder.encode({:class => 'MyClass',:args => []}),mock()) }
 
     it 'should start the fetcher when starting the manager' do
       fetcher.should_receive(:start)
@@ -26,14 +28,17 @@ describe Chore::Manager do
     end
 
     describe 'assigning messages' do
-      it 'should create a worker if one is available' do
-        worker = mock()
-        Chore::Worker.should_receive(:new).with(work).and_return(worker)
+      let(:worker) { mock() }
+
+      before(:each) do
         worker.should_receive(:start).with()
+      end
+
+      it 'should create a worker if one is available' do
+        Chore::Worker.should_receive(:new).with(work,{}).and_return(worker)
         manager.assign(work)
       end
     end
-
   end
 
 end

data/spec/chore/queues/sqs/consumer_spec.rb

@@ -4,10 +4,11 @@ describe Chore::Queues::SQS::Consumer do
   let(:queue_name) { "test" }
   let(:queue_url) { "test_url" }
   let(:queues) { double("queues") }
-  let(:queue) { double("test_queue") }
+  let(:queue) { double("test_queue", :visibility_timeout=>10, :url=>"test_queue", :name=>"test_queue") }
   let(:options) { {} }
   let(:consumer) { Chore::Queues::SQS::Consumer.new(queue_name) }
-  let(:message) { TestMessage.new("handle",queue_name,"message body", 1) }
+  let(:message) { TestMessage.new("handle",queue, "message body", 1) }
+  let(:message_data) {{:id=>message.id, :queue=>message.queue.url, :visibility_timeout=>message.queue.visibility_timeout}}
   let(:pool) { double("pool") }
   let(:sqs) { double('AWS::SQS') }
 
@@ -18,7 +19,6 @@ describe Chore::Queues::SQS::Consumer do
     queues.stub(:url_for) { queue_url }
     queues.stub(:[]) { queue }
     queue.stub(:receive_message) { message }
-    queue.stub(:visibility_timeout) { 10 }
     pool.stub(:empty!) { nil }
   end
 
@@ -72,7 +72,7 @@ describe Chore::Queues::SQS::Consumer do
     end
 
     it "should check the uniqueness of the message" do
-      Chore::DuplicateDetector.any_instance.should_receive(:found_duplicate?).with(message).and_return(false)
+      Chore::DuplicateDetector.any_instance.should_receive(:found_duplicate?).with(message_data).and_return(false)
       consumer.consume
     end
 
@@ -81,7 +81,7 @@ describe Chore::Queues::SQS::Consumer do
     end
 
     it 'should not yield for a dupe message' do
-      Chore::DuplicateDetector.any_instance.should_receive(:found_duplicate?).with(message).and_return(true)
+      Chore::DuplicateDetector.any_instance.should_receive(:found_duplicate?).with(message_data).and_return(true)
       expect {|b| consumer.consume(&b) }.not_to yield_control
     end
 

data/spec/chore/strategies/worker/forked_worker_strategy_spec.rb

@@ -9,7 +9,7 @@ describe Chore::Strategy::ForkedWorkerStrategy do
     strategy
   end
   let(:job_timeout) { 60 }
-  let(:job) { Chore::UnitOfWork.new(SecureRandom.uuid, 'test', job_timeout, Chore::JsonEncoder.encode(TestJob.job_hash([1,2,"3"])), 0) }
+  let(:job) { Chore::UnitOfWork.new(SecureRandom.uuid, 'test', job_timeout, Chore::Encoder::JsonEncoder.encode(TestJob.job_hash([1,2,"3"])), 0) }
   let!(:worker) { Chore::Worker.new(job) }
   let(:pid) { Random.rand(2048) }
 
@@ -41,7 +41,7 @@ describe Chore::Strategy::ForkedWorkerStrategy do
     end
 
     it 'should assign a job to a new worker' do
-      Chore::Worker.should_receive(:new).with(job).and_return(worker)
+      Chore::Worker.should_receive(:new).with(job, {}).and_return(worker)
       worker.should_receive(:start)
       forker.assign(job)
     end

data/spec/chore/worker_spec.rb

@@ -15,27 +15,36 @@ describe Chore::Worker do
   let(:job_args) { [1,2,'3'] }
   let(:job) { SimpleJob.job_hash(job_args) }
 
-  it 'should use a default encoder' do
+  it 'should use a default payload handler' do
     worker = Chore::Worker.new
-    worker.options[:encoder].should == Chore::JsonEncoder
+    worker.options[:payload_handler].should == Chore::Job
   end
 
-  it 'should process a single job' do
-    work = Chore::UnitOfWork.new('1', 'test', 60, Chore::JsonEncoder.encode(job), 0, consumer)
-    SimpleJob.should_receive(:perform).with(*job_args)
-    consumer.should_receive(:complete).with('1')
-    w = Chore::Worker.new(work)
-    w.start
-  end
+  shared_examples_for "a worker" do
+    it 'processing a single job' do
+      work = Chore::UnitOfWork.new('1', 'test', 60, encoded_job, 0, consumer)
+      SimpleJob.should_receive(:perform).with(*payload)
+      consumer.should_receive(:complete).with('1')
+      w = Chore::Worker.new(work, {:payload_handler => payload_handler})
+      w.start
+    end
 
-  it 'should process multiple jobs' do
-    work = []
-    10.times do |i|
-      work << Chore::UnitOfWork.new(i, 'test', 60, Chore::JsonEncoder.encode(job), 0, consumer)
+    it 'processing multiple jobs' do
+      work = []
+      10.times do |i|
+        work << Chore::UnitOfWork.new(i, 'test', 60, encoded_job, 0, consumer)
+      end
+      SimpleJob.should_receive(:perform).exactly(10).times
+      consumer.should_receive(:complete).exactly(10).times
+      Chore::Worker.start(work, {:payload_handler => payload_handler})
     end
-    SimpleJob.should_receive(:perform).exactly(10).times
-    consumer.should_receive(:complete).exactly(10).times
-    Chore::Worker.start(work)
+  end
+
+  describe "with default payload handler" do
+    let(:encoded_job) { Chore::Encoder::JsonEncoder.encode(job) }
+    let(:payload_handler) { Chore::Job }
+    let(:payload) {job_args}
+    it_behaves_like "a worker"
   end
 
   describe 'expired?' do
@@ -43,7 +52,7 @@ describe Chore::Worker do
     let(:queue_timeouts) { [10, 20, 30] }
     let(:work) do
       queue_timeouts.map do |queue_timeout|
-        Chore::UnitOfWork.new('1', 'test', queue_timeout, Chore::JsonEncoder.encode(job), 0, consumer)
+        Chore::UnitOfWork.new('1', 'test', queue_timeout, Chore::Encoder::JsonEncoder.encode(job), 0, consumer)
       end
     end
     let(:worker) do
@@ -100,7 +109,7 @@ describe Chore::Worker do
     end
 
     context 'on perform' do
-      let(:encoded_job) { Chore::JsonEncoder.encode(job) }
+      let(:encoded_job) { Chore::Encoder::JsonEncoder.encode(job) }
       let(:parsed_job) { JSON.parse(encoded_job) }
 
       before(:each) do

data/spec/spec_helper.rb

@@ -26,11 +26,19 @@ class FakePublisher < Chore::Publisher
   end
 end
 
-TestMessage = Struct.new(:handle,:queue_name,:body,:receive_count) do
+TestMessage = Struct.new(:handle,:queue,:body,:receive_count) do
   def empty?
     false
   end
 
+  def queue_name
+    self.queue.name
+  end
+
+  def id
+    self.handle
+  end
+
   # Structs define a to_a behavior that is not compatible with array splatting. Remove it so that
   # [*message] on a struct will behave the same as on a string.
   undef_method :to_a
@@ -38,6 +46,7 @@ end
 
 
 RSpec.configure do |config|
+  config.include Chore::Util
   config.before do
     Chore.configure do |c|
       c.aws_access_key = ""

metadata

@@ -1,103 +1,103 @@
 --- !ruby/object:Gem::Specification
 name: chore-core
 version: !ruby/object:Gem::Version
-  version: 1.5.2
+  version: 1.5.10
 platform: ruby
 authors:
 - Tapjoy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-20 00:00:00.000000000 Z
+date: 2015-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.12'
-    - - ! '>='
+        version: '1.56'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.12.0
+        version: 1.56.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.12'
-    - - ! '>='
+        version: '1.56'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 1.12.0
+        version: 1.56.0
 - !ruby/object:Gem::Dependency
   name: thread
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.12.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.12.0
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '3.12'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Job processing with pluggable backends and strategies
@@ -120,10 +120,10 @@ files:
 - lib/chore/configuration.rb
 - lib/chore/consumer.rb
 - lib/chore/duplicate_detector.rb
+- lib/chore/encoders/json_encoder.rb
 - lib/chore/fetcher.rb
 - lib/chore/hooks.rb
 - lib/chore/job.rb
-- lib/chore/json_encoder.rb
 - lib/chore/manager.rb
 - lib/chore/publisher.rb
 - lib/chore/queues/filesystem/consumer.rb
@@ -147,10 +147,10 @@ files:
 - spec/chore/cli_spec.rb
 - spec/chore/consumer_spec.rb
 - spec/chore/duplicate_detector_spec.rb
+- spec/chore/encoders/json_encoder_spec.rb
 - spec/chore/fetcher_spec.rb
 - spec/chore/hooks_spec.rb
 - spec/chore/job_spec.rb
-- spec/chore/json_encoder_spec.rb
 - spec/chore/manager_spec.rb
 - spec/chore/queues/filesystem/filesystem_consumer_spec.rb
 - spec/chore/queues/sqs/consumer_spec.rb
@@ -176,17 +176,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Job processing... for the future!

data/lib/chore/json_encoder.rb

@@ -1,18 +0,0 @@
-require 'json'
-
-module Chore
-  # Json encoding for serializing jobs. 
-  module JsonEncoder
-    class << self
-      # Encodes the +job+ into JSON using the standard ruby JSON parsing library
-      def encode(job)
-        JSON.generate(job.to_hash)
-      end
-
-      # Decodes the +job+ from JSON into a ruby Hash using the standard ruby JSON parsing library
-      def decode(job)
-        JSON.parse(job)
-      end
-    end
-  end
-end