proletariat 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f18e4d2458922ab3466f6913a079616c7ab3e5ab
-  data.tar.gz: 0ed0637283e0056fca8e260bbb4fe97dc26eca1e
+  metadata.gz: 857a43f0570e09ed05e46f910b5a746474116604
+  data.tar.gz: 4c5a325c9d11bf52e1ac69b5c1c365a291ecfcc7
 SHA512:
-  metadata.gz: f6e31240a83223f4d67f2f0e08ae194927f07b6671fa193b32e7ed6527d37e66de29ff7830047cbbeb23a300b19eb43b1ad5765546ac51f406bdabd0d83cecee
-  data.tar.gz: 431900e4e0b1f59427666ceeca3197297c3723441c0f472242ee40dda0e3afe148bbc63cde95d10818a65358130a3ea83e46006042696385d6f4d7bd6a07dd3a
+  metadata.gz: 00801d41f5a02e885f82a15bcf4d189e3f5ebe80892afe982cec5efe2f3effe3dbbe3e1aca7e8477110907045968bb991ea83ddeac1f15ef844eaba287428049
+  data.tar.gz: b13a39c9cb321ade6df9a2566d00942edd1ddb35339cae3102e2b1d33279a11f585e8aeef0e7a494cb7334d08255b5308c03e4f748fbef9b0a5b632c56937486

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.0.5
+
+Features:
+
+  - Added access to message headers in #work (breaking change).
+  - Added exponential back-off for failing workers.
+
 ## 0.0.4
 
 Features:

data/README.md

@@ -41,7 +41,7 @@ Here's a complete example:
     class SendUserIntroductoryEmail < Proletariat::Worker
       listen_on 'user.created'
 
-      def work(message, key)
+      def work(message, key, headers)
         params = JSON.parse(message)
 
         UserMailer.introductory_email(params).deliver!

data/lib/proletariat/configuration.rb

@@ -4,6 +4,9 @@ module Proletariat
     # Public: The default name used for the RabbitMQ topic exchange.
     DEFAULT_EXCHANGE_NAME = 'proletariat'
 
+    # Public: The default maximum seconds to delay a failed job requeue.
+    DEFAULT_MAX_RETRY_DELAY = (ENV['MAX_RETRY_DELAY'] || 60).to_i
+
     # Public: The default number of threads to use for publishers.
     DEFAULT_PUBLISHER_THREADS = (ENV['PUBLISHER_THREADS'] || 2).to_i
 
@@ -19,6 +22,9 @@ module Proletariat
     # Internal: Sets the logger.
     attr_writer :logger
 
+    # Internal: Sets the maximum seconds to delay a failed job requeue.
+    attr_writer :max_retry_delay
+
     # Internal: Sets the number of threads to use for publishers.
     attr_writer :publisher_threads
 
@@ -67,6 +73,13 @@ module Proletariat
       @logger ||= Logger.new(STDOUT)
     end
 
+    # Public: Returns the set max delay seconds or a default.
+    #
+    # Returns a Fixnum.
+    def max_retry_delay
+      @max_retry_delay ||= DEFAULT_MAX_RETRY_DELAY
+    end
+
     # Public: Returns the set number of publisher threads or a default.
     #
     # Returns a Fixnum.

data/lib/proletariat/publisher.rb

@@ -44,10 +44,12 @@ module Proletariat
     #           with the RabbitMQ convention you can use the '*' character to
     #           replace one word and the '#' to replace many words.
     # message - The message as a String.
+    # headers - Hash of message headers.
     #
     # Returns nil.
-    def work(to, message)
-      exchange.publish message, routing_key: to, persistent: true
+    def work(to, message, headers)
+      exchange.publish message, routing_key: to, persistent: true,
+                                headers: headers
 
       nil
     end

data/lib/proletariat/runner.rb

@@ -25,10 +25,11 @@ module Proletariat
     #           with the RabbitMQ convention you can use the '*' character to
     #           replace one word and the '#' to replace many words.
     # message - The message as a String.
+    # headers - Hash of message headers.
     #
     # Returns nil.
-    def publish(to, message)
-      supervisor['publishers'].post to, message
+    def publish(to, message = '', headers = {})
+      supervisor['publishers'].post to, message, headers
 
       nil
     end

data/lib/proletariat/subscriber.rb

@@ -58,6 +58,8 @@ module Proletariat
         acknowledger.acknowledge_on_channel channel
         acknowledgers.delete acknowledger
       end
+
+      completed_retries.each { |r| scheduled_retries.delete r }
     end
 
     # Public: Purge the RabbitMQ queue.
@@ -102,6 +104,39 @@ module Proletariat
       nil
     end
 
+    # Internal: Get scheduled retries whose messages have been requeued.
+    #
+    # Returns an Array of Retrys.
+    def completed_retries
+      scheduled_retries.select { |r| r.requeued? }
+    end
+
+    # Internal: Forwards all message bodies to listener#post. Auto-acks
+    #           messages not meant for this subscriber's workers.
+    #
+    # Returns nil.
+    def handle_message(info, properties, body)
+      if handles_worker_type? properties.headers['worker']
+        future = listener.post?(body, info.routing_key, properties.headers)
+        acknowledgers << Acknowledger.new(future, info.delivery_tag, {
+          message: body, key: info.routing_key, headers: properties.headers,
+          worker:  queue_config.queue_name }, scheduled_retries)
+      else
+        channel.acknowledge info.delivery_tag
+      end
+
+      nil
+    end
+
+    # Internal: Checks if subscriber should handle message for given worker
+    #           header.
+    #
+    # Returns true if should be handled or header is nil.
+    # Returns false if should not be handled.
+    def handles_worker_type?(worker_header)
+      [nil, queue_config.queue_name].include? worker_header
+    end
+
     # Internal: Get acknowledgers for messages whose work has completed.
     #
     # Returns an Array of Acknowledgers.
@@ -111,14 +146,18 @@ module Proletariat
       end
     end
 
+    def scheduled_retries
+      @scheduled_retries ||= []
+    end
+
     # Internal: Starts a consumer on the queue. The consumer forwards all
-    #           message bodies to listener#post.
+    #           message bodies to listener#post. Auto-acks messages not meant
+    #           for this subscriber's workers.
     #
     # Returns nil.
     def start_consumer
       @consumer = bunny_queue.subscribe ack: true do |info, properties, body|
-        future = listener.post?(body, info.routing_key)
-        acknowledgers << Acknowledger.new(future, info.delivery_tag)
+        handle_message info, properties, body
 
         nil
       end
@@ -133,6 +172,7 @@ module Proletariat
     def stop_consumer
       @consumer.cancel if @consumer
       wait_for_acknowledgers if acknowledgers.any?
+      scheduled_retries.each { |r| r.expedite }
 
       nil
     end
@@ -160,11 +200,15 @@ module Proletariat
 
       # Public: Creates a new Acknowledger instance.
       #
-      # future       - A future-like object holding the Worker response.
-      # delivery_tag - The RabbitMQ delivery tag to be used when ack/nacking.
-      def initialize(future, delivery_tag)
-        @future       = future
-        @delivery_tag = delivery_tag
+      # future            - A future-like object holding the Worker response.
+      # delivery_tag      - The RabbitMQ delivery tag for ack/nacking.
+      # properties        - The original message properties; for requeuing.
+      # scheduled_retries - An Array to hold any created Retrys.
+      def initialize(future, delivery_tag, properties, scheduled_retries)
+        @future            = future
+        @delivery_tag      = delivery_tag
+        @properties        = properties
+        @scheduled_retries = scheduled_retries
       end
 
       # Public: Retrieves the value from the future and sends the relevant
@@ -232,7 +276,9 @@ module Proletariat
       # Returns nil.
       def acknowledge_error(channel)
         Proletariat.logger.error future.reason
-        channel.reject delivery_tag, true
+
+        scheduled_retries << Retry.new(properties)
+        channel.acknowledge delivery_tag
 
         nil
       end
@@ -242,6 +288,85 @@ module Proletariat
 
       # Internal: Returns the future-like object holding the Worker response.
       attr_reader :future
+
+      # Internal: Returns the original message properties.
+      attr_reader :properties
+
+      # Internal: Returns the Array of Retrys.
+      attr_reader :scheduled_retries
+
+      # Internal: Used publish an exponential delayed requeue for failures.
+      class Retry
+        # Public: Creates a new Retry instance. Sets appropriate headers for
+        #         requeue message.
+        #
+        # properties - The original message properties.
+        def initialize(properties)
+          @properties = properties
+
+          properties[:headers]['failures'] = failures
+          properties[:headers]['worker']   = properties[:worker]
+
+          @scheduled_task = Concurrent::ScheduledTask.new(retry_delay) do
+            requeue_message
+          end
+        end
+
+        # Public: Attempt to requeue the message immediately if pending or
+        #         wait for natural completion.
+        #
+        # Returns nil.
+        def expedite
+          if scheduled_task.cancel
+            requeue_message
+          else
+            scheduled_task.value
+          end
+
+          nil
+        end
+
+        # Public: Tests whether the message has been requeued.
+        #
+        # Returns a Boolean.
+        def requeued?
+          scheduled_task.fulfilled?
+        end
+
+        private
+
+        # Internal: Returns the original message properties.
+        attr_reader :properties
+
+        # Internal: Returns the ScheduledTask which will requeue the message.
+        attr_reader :scheduled_task
+
+        # Internal: Fetches the current number of message failures from the
+        #           headers. Defaults to 1.
+        #
+        # Returns a Fixnum.
+        def failures
+          @failures ||= (properties[:headers]['failures'] || 0) + 1
+        end
+
+        # Internal: Performs the actual message requeue.
+        #
+        # Returns nil.
+        def requeue_message
+          Proletariat.publish(properties[:key], properties[:message],
+                              properties[:headers])
+
+          nil
+        end
+
+        # Internal: Calculates an exponential retry delay based on the previous
+        #           number of failures. Capped with configuration setting.
+        #
+        # Returns the delay in seconds as a Fixnum.
+        def retry_delay
+          [2**failures, Proletariat.max_retry_delay].min
+        end
+      end
     end
   end
 end

data/lib/proletariat/testing/expectation_guarantor.rb

@@ -125,10 +125,12 @@ module Proletariat
         # Public: Handles message calls from a subscriber and increments the
         #         count. Return value matches interface expected by Subscriber.
         #
-        # message - The contents of the message.
+        # message     - The contents of the message.
+        # routing_key - Routing key for messages.
+        # headers     - Hash of message headers.
         #
         # Returns a future-like object holding an :ok Symbol.
-        def post?(message, routing_key)
+        def post?(message, routing_key, headers)
           self.count = count + 1
 
           Concurrent::Future.new { :ok }

data/lib/proletariat/version.rb

@@ -1,4 +1,4 @@
 # Public: Adds a constant for the current version number.
 module Proletariat
-  VERSION = '0.0.4'
+  VERSION = '0.0.5'
 end

data/lib/proletariat/worker.rb

@@ -38,7 +38,7 @@ module Proletariat
     # message - The incoming message.
     #
     # Raises NotImplementedError unless implemented in subclass.
-    def work(message, routing_key)
+    def work(message, routing_key, headers)
       fail NotImplementedError
     end
 
@@ -71,11 +71,12 @@ module Proletariat
     #           with the RabbitMQ convention you can use the '*' character to
     #           replace one word and the '#' to replace many words.
     # message - The message as a String.
+    # headers - Hash of message headers.
     #
     # Returns nil.
-    def publish(to, message = '')
+    def publish(to, message = '', headers = {})
       log "Publishing to: #{to}"
-      Proletariat.publish to, message
+      Proletariat.publish to, message, headers
 
       nil
     end

data/spec/lib/proletariat_spec.rb

@@ -8,7 +8,7 @@ class PingWorker < Proletariat::Worker
     attr_accessor :pinged
   end
 
-  def work(message, routing_key)
+  def work(message, routing_key, headers)
     self.class.pinged = true
 
     log 'PING'
@@ -24,10 +24,15 @@ class PongWorker < Proletariat::Worker
   listen_on :pong
 
   class << self
+    attr_accessor :fail_mode
     attr_accessor :ponged
   end
 
-  def work(message, routing_key)
+  def work(message, routing_key, headers)
+    if self.class.fail_mode == true
+      fail 'Error' unless headers['failures'] == 2
+    end
+
     self.class.ponged = true
 
     log 'PONG'
@@ -39,18 +44,38 @@ class PongWorker < Proletariat::Worker
 end
 
 describe Proletariat do
-  it 'should roughly work' do
+  before do
     Proletariat.configure do
       config.logger         = Logger.new('/dev/null')
       config.worker_classes = [PingWorker, PongWorker]
     end
 
+    PongWorker.fail_mode = false
+
+    Proletariat.purge
     Proletariat.run!
     sleep 2
-    Proletariat.publish 'ping', ''
-    sleep 3
+  end
+
+  after do
     Proletariat.stop
     Proletariat.purge
+    PingWorker.pinged = false
+    PongWorker.ponged = false
+  end
+
+  it 'should roughly work' do
+    Proletariat.publish 'ping', ''
+    sleep 5
+
+    expect(PingWorker.pinged).to be_truthy
+    expect(PongWorker.ponged).to be_truthy
+  end
+
+  it 'should work in error conditions' do
+    PongWorker.fail_mode = true
+    Proletariat.publish 'ping', ''
+    sleep 10
 
     expect(PingWorker.pinged).to be_truthy
     expect(PongWorker.ponged).to be_truthy

data/spec/lib/testing/expectation_guarantor_spec.rb

@@ -34,19 +34,19 @@ module Proletariat
 
         it 'should increment the count' do
           counter = ExpectationGuarantor::MessageCounter.new(1)
-          counter.post?('message', 'key')
+          counter.post?('message', 'key', {})
           expect(counter.expected_messages_received?).to be_truthy
         end
 
         it 'should return a future containing :ok' do
           counter = ExpectationGuarantor::MessageCounter.new(1)
           expect(Concurrent::Future).to receive(:new)
-          counter.post?('message', 'key')
+          counter.post?('message', 'key', {})
         end
 
         it 'should ensure the returned future contains :ok' do
           counter = ExpectationGuarantor::MessageCounter.new(1)
-          future = counter.post?('message', 'key')
+          future = counter.post?('message', 'key', {})
           expect(future.block.call).to eq :ok
         end
       end

data/spec/lib/worker_spec.rb

@@ -31,7 +31,7 @@ module Proletariat
 
     describe '#work' do
       it 'should raise NotImplementedError' do
-        expect { Worker.new.work('message', 'key') }.to \
+        expect { Worker.new.work('message', 'key', {}) }.to \
           raise_exception NotImplementedError
       end
     end
@@ -53,12 +53,12 @@ module Proletariat
 
     describe '#publish' do
       it 'should forward the message to the publisher' do
-        expect(Proletariat).to receive(:publish).with('topic', 'message')
-        Worker.new.publish 'topic', 'message'
+        expect(Proletariat).to receive(:publish).with('topic', 'message', {})
+        Worker.new.publish 'topic', 'message', {}
       end
 
       it 'should have a blank default message' do
-        expect(Proletariat).to receive(:publish).with('topic', '')
+        expect(Proletariat).to receive(:publish).with('topic', '', {})
         Worker.new.publish 'topic'
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proletariat
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Sebastian Edwards
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-17 00:00:00.000000000 Z
+date: 2014-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec