dispatch-rider 1.0.3 → 1.1.4

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    YmRhMWRiODkzNDMzMDU5NzBjYjNlYzFjZGE3Yzk5ZGFlNDIxMDViOA==
-  data.tar.gz: !binary |-
-    ODRkMGIyZWUzODdhMmYxNDg0MTgzZjQxYmQ3OGJlYWFkZTE1YTk0ZQ==
+SHA1:
+  metadata.gz: 5abe7d067b30a87f2e255eae8d4cfc94782567b7
+  data.tar.gz: 2044e59ff77593e6c0d0aa10399210066228a7fd
 SHA512:
-  metadata.gz: !binary |-
-    ZTUzNjVhNDVkNjY0N2VjY2NmNjAxZDQ3NDk0NWZhNTY1M2ZjODA1ZjA0MGZi
-    YzVjNmFjZWM1MWQ0OGZjNzhiMDM3MzdmOWZjMDMzOGUyYjRlMzU1YTI5YmM5
-    YzZlMzRmNzY1MWZhYjE5NWM3MDdiNGMyNWUxZmVhYTZhZTkyZTk=
-  data.tar.gz: !binary |-
-    OGJjYjBkYWY5NzdlOTIxMTgyYzU4ZTI2ODc3ODgzNzM2MmE0YjMzNGJmMDI2
-    OWM5OWQ1MTBiNmNjNzg2M2I4ZTVlODY5NTE1MGM5NzJjYmExYzMzYThmODc2
-    ODBlZGI4YWNlYTUwMTU3MzI5MDY1MTg3ZjQzYmE3MWI4YWQ5MmM=
+  metadata.gz: 36e44305e710150417a6bd3f3673ff8b9fd33cb8ac44e259114eebfdb3f9d8690b270b16e03942978ea8bb4d9af2679111736d03923e9c173a3a9b88a43f9120
+  data.tar.gz: 336fbb538b2a9998cb3e53e26217b3dedaad6e198f25ee02e79f96513b102045b38b9881b32ffad529a0a6295f42789325bb5cb9ca8ee4a4767ad7658ac3fbbc

data/README.md

@@ -212,6 +212,7 @@ DispatchRider.config do |config|
   end
 
   config.logger = Rails.logger
+  config.default_retry_timeout = 300
 
   config.error_handler = DispatchRider::DefaultErrorHandler # an object that responds to .call(message, exception)
 
@@ -243,6 +244,9 @@ Each callback can have hooks plugged into it at `before`, `after` and `around` t
 ### Manual Setup
 
 To setup a subscriber you'll need message handlers. The handlers are named the same as the message subjects.
+Each handler may also specify a retry_timeout as shown below.  When a job throws an exception it will be put back
+on the queue in that time period if the queue supports timeouts.  If the underlying queue (such as filesystem) does
+not support retry then this setting is ineffective.
 
 Sample message handler:
 ```ruby
@@ -253,6 +257,33 @@ class ReadNews < DispatchRider::Handlers::Base
       puts headline
     end
   end
+  
+  def retry_timeout
+    10.minutes
+  end
+end
+```
+
+### Timeout & retry handling
+
+If you have a long running job, or if you wish to retry a job later, you may use two methods in your
+handler class.  return_to_queue and extend_timeout.
+
+return_to_queue will retry your item immediately.
+extend_timeout will tell the queue you wish to hold this item longer.
+
+```ruby
+# app/handlers/foo_handler
+class LongRunning < DispatchRider::Handlers::Base
+  def process(body)
+    my_loop.each do |item|
+      
+      #... do some work ...
+      extend_timeout(1.hour)
+    end
+  rescue OutOfResourcesImOutError
+    return_to_queue #oops!  Better give this to somebody else!
+  end
 end
 ```
 

data/lib/dispatch-rider/configuration.rb

@@ -14,6 +14,10 @@ module DispatchRider
     end
 
     delegate :before, :after, :around, :to => :callbacks
+    
+    def default_retry_timeout=(val)
+      DispatchRider::Handlers::Base.set_default_retry(val)
+    end
 
     def handlers
       @handlers ||= begin

data/lib/dispatch-rider/dispatcher.rb

@@ -19,7 +19,7 @@ module DispatchRider
 
     def dispatch(message)
       callbacks.invoke(:dispatch_message, message) do
-        handler_registrar.fetch(message.subject).new.do_process(message.body)
+        handler_registrar.fetch(message.subject).new.do_process(message)
       end
       true # success => true (delete message)
     end

data/lib/dispatch-rider/handlers/base.rb

@@ -3,16 +3,54 @@ module DispatchRider
     class Base
       include NamedProcess
       extend InheritanceTracking
+      
+      class << self
+        def set_default_retry( amount ) 
+          define_method(:retry_timeout) do
+            amount
+          end
+        end
+      end
+      
+      attr_reader :raw_message
 
-      def do_process(options)
+      def do_process(raw_message)
         with_named_process(self.class.name) do
-          process(options)
+          @raw_message = raw_message
+          process(raw_message.body)
         end
+      rescue Exception => e
+        self.retry if self.retry_on_failure?
+        raise e
       end
 
-      def process(options)
+      def process(message)
         raise NotImplementedError, "Method 'process' not overridden in subclass!"
       end
+      
+      protected
+      
+      def extend_timeout(timeout)
+        raw_message.extend_timeout(timeout)
+      end
+      
+      def return_to_queue
+        raw_message.return_to_queue
+      end
+      
+      def retry
+        timeout = retry_timeout
+        case timeout
+        when :immediate
+          return_to_queue
+        else
+          extend_timeout(timeout)
+        end
+      end
+      
+      def retry_on_failure?
+        self.respond_to? :retry_timeout
+      end
     end
   end
 end

data/lib/dispatch-rider/queue_services/aws_sqs.rb

@@ -4,6 +4,7 @@ module DispatchRider
   module QueueServices
     class AwsSqs < Base
       require "dispatch-rider/queue_services/aws_sqs/message_body_extractor"
+      require "dispatch-rider/queue_services/aws_sqs/sqs_received_message"
 
       class AbortExecution < RuntimeError; end
       class VisibilityTimeoutExceeded < RuntimeError; end
@@ -26,11 +27,11 @@ module DispatchRider
       def pop(&block)
         begin
           queue.receive_message do |raw_item|
-            obj = ::OpenStruct.new(:item => raw_item, :message => construct_message_from(raw_item))
+            obj = SqsReceivedMessage.new(construct_message_from(raw_item), raw_item, queue)
 
-            visibility_timout_shield(obj.message) do
-              raise AbortExecution, "false received from handler" unless block.call(obj.message)
-              obj.message
+            visibility_timeout_shield(obj) do
+              raise AbortExecution, "false received from handler" unless block.call(obj)
+              obj
             end
 
           end
@@ -57,17 +58,12 @@ module DispatchRider
 
       private
 
-      def visibility_timeout
-        queue.visibility_timeout
-      end
-
-      def visibility_timout_shield(message)
-        start_time = Time.now
-        timeout = visibility_timeout # capture it at start
+      def visibility_timeout_shield(message)
         begin
           yield
         ensure
-          duration = Time.now - start_time
+          duration = Time.now - message.start_time
+          timeout = message.total_timeout
           raise VisibilityTimeoutExceeded, "message: #{message.subject}, #{message.body.inspect} took #{duration} seconds while the timeout was #{timeout}" if duration > timeout
         end
       end

data/lib/dispatch-rider/queue_services/aws_sqs/sqs_received_message.rb

@@ -0,0 +1,34 @@
+module DispatchRider
+  module QueueServices
+    class AwsSqs < Base
+      class SqsReceivedMessage < ReceivedMessage
+        attr_reader :total_timeout, :start_time
+        
+        def initialize(message, raw_item, queue)
+          @total_timeout = queue.visibility_timeout
+          @start_time = Time.now
+          super(message, raw_item)
+        end
+        
+        # NOTE: Setting the visibility timeout resets the timeout to NOW and makes it visibility timeout this time
+        # Essentially resetting the timer on this message
+        def extend_timeout(timeout)
+          item.visibility_timeout = timeout
+          if timeout > 0
+            @total_timeout = timeout + (Time.now - start_time).to_i
+          end
+        end
+        
+        # We effectively return the item to the queue by setting
+        # the visibility timeout to zero.  The item
+        # should become immediately visible.
+        # The next receiver will reset the visibility
+        # to something appropriate
+        def return_to_queue
+          extend_timeout(0)
+        end
+        
+      end
+    end
+  end
+end

data/lib/dispatch-rider/queue_services/base.rb

@@ -5,6 +5,7 @@
 # initialize, push, pop and empty?
 module DispatchRider
   module QueueServices
+    require "dispatch-rider/queue_services/received_message"
     class Base
       attr_accessor :queue
 
@@ -26,17 +27,23 @@ module DispatchRider
         raise NotImplementedError
       end
 
+      
+      #If you pass a block into pop it will wrap the deletion of the message with it's handling
       def pop(&block)
-        obj = head
-        if obj
-          block.call(obj.message) && delete(obj.item)
-          obj.message
+        received = head
+        if received
+          block.call(received) && delete(received.item)
+          received
         end
       end
 
       def head
         raw_item = raw_head
-        raw_item && ::OpenStruct.new(:item => raw_item, :message => construct_message_from(raw_item))
+        raw_item && received_message_for(raw_item)
+      end
+      
+      def received_message_for(raw_item)
+         QueueServices::ReceivedMessage.new(construct_message_from(raw_item), raw_item)
       end
 
       def raw_head

data/lib/dispatch-rider/queue_services/file_system.rb

@@ -22,10 +22,18 @@ module DispatchRider
       def raw_head
         queue.pop
       end
+      
+      def received_message_for(raw_item)
+         FsReceivedMessage.new(construct_message_from(raw_item), raw_item, queue)
+      end
 
       def construct_message_from(item)
         deserialize(item.read)
       end
+      
+      def put_back(item)
+        queue.put_back(item)
+      end
 
       def delete(item)
         queue.remove item

data/lib/dispatch-rider/queue_services/file_system/fs_received_message.rb

@@ -0,0 +1,23 @@
+module DispatchRider
+  module QueueServices
+    class FileSystem < Base
+      class FsReceivedMessage < ReceivedMessage
+        attr_reader :queue
+        
+        def initialize(message, item, queue)
+          @queue = queue
+          super(message, item)
+        end
+        
+        def extend_timeout(timeout)
+          #file system doesn't support timeouts on items, so we ignore this.
+        end
+        
+        def return_to_queue
+          queue.put_back(item)
+        end
+        
+      end
+    end
+  end
+end

data/lib/dispatch-rider/queue_services/file_system/queue.rb

@@ -21,6 +21,11 @@ module DispatchRider
           FileUtils.mv(file_path, file_path_inflight)
           File.new(file_path_inflight)
         end
+        
+        def put_back(item)
+          add(item)
+          remove(item)
+        end
 
         def remove(item)
           item.close

data/lib/dispatch-rider/queue_services/received_message.rb

@@ -0,0 +1,23 @@
+module DispatchRider
+  module QueueServices
+    class ReceivedMessage < SimpleDelegator
+      
+      #Item is the raw message item as returned by the queue implementor
+      #it's contents will depend on the queue being used
+      attr_reader :item
+      
+      def initialize(message, item)
+        @item = item
+        super(message)
+      end
+      
+      def extend_timeout(time)
+        raise NotImplementedError
+      end
+      
+      def return_to_queue
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/dispatch-rider/version.rb

@@ -1,4 +1,4 @@
 # This file specifies the current version of the gem.
 module DispatchRider
-  VERSION = "1.0.3"
+  VERSION = "1.1.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dispatch-rider
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.1.4
 platform: ruby
 authors:
 - Suman Mukherjee
@@ -11,62 +11,62 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-15 00:00:00.000000000 Z
+date: 2014-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: activemodel
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: daemons
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -87,56 +87,56 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: travis-lint
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: debugger
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: Messaging system that is customizable based on which queueing system
@@ -183,9 +183,12 @@ files:
 - lib/dispatch-rider/queue_services.rb
 - lib/dispatch-rider/queue_services/aws_sqs.rb
 - lib/dispatch-rider/queue_services/aws_sqs/message_body_extractor.rb
+- lib/dispatch-rider/queue_services/aws_sqs/sqs_received_message.rb
 - lib/dispatch-rider/queue_services/base.rb
 - lib/dispatch-rider/queue_services/file_system.rb
+- lib/dispatch-rider/queue_services/file_system/fs_received_message.rb
 - lib/dispatch-rider/queue_services/file_system/queue.rb
+- lib/dispatch-rider/queue_services/received_message.rb
 - lib/dispatch-rider/queue_services/simple.rb
 - lib/dispatch-rider/registrars.rb
 - lib/dispatch-rider/registrars/base.rb
@@ -213,17 +216,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.11
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: Messaging system based on the reactor patter. You can publish messages to