leveret 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ad5d86c738d44521e7c94d660a85cd7146c6934
-  data.tar.gz: 4df57359cff0b01b6dd58d71b0b5a10962e605c6
+  metadata.gz: c09edf84323e1f658a68c1a63e642fbe6a24570e
+  data.tar.gz: d81b7f5a2637e8a3b6831db528a0aed27b17458e
 SHA512:
-  metadata.gz: 4bcc8fe946aa915c5ab765a6b8cd99ead0e2254ccc88ac4852de110543d7afff7465674d5facf604c7addff393c52cea17c36b38869873778bce864d0264825e
-  data.tar.gz: 1c2033a0b4b61b3147551744f67f4e8a09dfedd09d6f66c9204a9e816f209ed285cc13fa4a08c9c5bb157b4ca7ddaedeb7cf395059e89c19e12834f259579fc4
+  metadata.gz: 94522b76698bc0bac4d171aad0b9a91a2ff9c7ded392539a78c450e97a4e910cc796258ca2f94d535a8df6a84eebe74f01dca445e5d093bec93fa5700b54eb6d
+  data.tar.gz: c5b6f7493a901b39a6abe11d9de9df0b3d05460915907cc534249220afec2766c2cebe4cc1d66ab9ed312ff9bc7c5e80d27e5a5f07d3acf93ecf56b4f4e1bf53

data/lib/leveret/configuration.rb

@@ -9,6 +9,16 @@ module Leveret
   # @!attribute queue_name_prefix
   #   @return [String] This value will be prefixed to all queues created on your RabbitMQ instance.
   #     Default: +"leveret_queue"+
+  # @!attribute delay_exchange_name
+  #   @return [String] Name of the exchange for Leveret to publish to that should be processed later.
+  #     Default: +"leveret_delay_exch"+
+  # @!attribute delay_queue_name
+  #   @return [String] Name of the queue for Leveret to use to store messages that should be processed later.
+  #     Default: +"leveret_delay_queue"+
+  # @!attribute delay_time
+  #   @return [Integer] Amount of time a delayed message should say on the delay queue in milliseconds Default: +10000+
+  # @!attribute delay_exchange_name
+  #   @return [String] Name of the exchange for Leveret to publish messages to. Default: +"leveret_exch"+
   # @!attribute log_file
   #   @return [String] The path where logfiles should be written to. Default: +STDOUT+
   # @!attribute log_level
@@ -24,7 +34,7 @@ module Leveret
   #   @return [Integer] The number of jobs that can be processes simultanously. Default: +1+
   class Configuration
     attr_accessor :amqp, :exchange_name, :queue_name_prefix, :log_file, :log_level, :default_queue_name, :after_fork,
-      :error_handler, :concurrent_fork_count
+      :error_handler, :concurrent_fork_count, :delay_exchange_name, :delay_queue_name, :delay_time
 
     # Create a new instance of Configuration with a set of sane defaults.
     def initialize
@@ -40,6 +50,9 @@ module Leveret
       self.log_level = Logger::DEBUG
       self.queue_name_prefix = 'leveret_queue'
       self.default_queue_name = 'standard'
+      self.delay_exchange_name = 'leveret_delay_exch'
+      self.delay_queue_name = 'leveret_delay_queue'
+      self.delay_time = 10_000
       self.after_fork = proc {}
       self.error_handler = proc { |ex| ex }
       self.concurrent_fork_count = 1

data/lib/leveret/delay_queue.rb

@@ -0,0 +1,37 @@
+module Leveret
+  # Connects to a special queue which keeps messages in a holding pattern until a timeout expires and then
+  # publishes those messages back to the main queue for processing.
+  class DelayQueue
+    extend Forwardable
+
+    attr_reader :queue
+
+    def_delegators :Leveret, :configuration, :channel, :log
+
+    def initialize
+      @queue = connect_to_queue
+    end
+
+    # Place a message onto the delay queue, which will later be expired and sent back to the main exchange
+    #
+    # @param [Message] A message received and processed already
+    def republish(message)
+      delay_exchange.publish(message.params.serialize, expiration: configuration.delay_time, persistent: true,
+        routing_key: message.routing_key, priority: message.priority)
+    end
+
+    private
+
+    def connect_to_queue
+      queue = channel.queue(configuration.delay_queue_name, durable: true,
+        arguments: { 'x-dead-letter-exchange': configuration.exchange_name })
+      queue.bind(delay_exchange)
+      log.info "Connected to #{configuration.delay_queue_name}, bound to #{configuration.delay_exchange_name}"
+      queue
+    end
+
+    def delay_exchange
+      @delay_exchange ||= channel.exchange(Leveret.configuration.delay_exchange_name, type: :fanout, durable: :true)
+    end
+  end
+end

data/lib/leveret/job.rb

@@ -36,14 +36,15 @@ module Leveret
   #   MyJob.enqueue(test_text: "Hi there, please write this different text to the file", queue_name: 'other_queue')
   #
   module Job
-    # Raise this when your job has failed, but try again when a worker is
-    # available again.
+    # Raise this when your job has failed, but try again as soon as another worker is available.
     class RequeueJob < StandardError; end
 
-    # Raise this when your job has failed, but you don't want to requeue it
-    # and try again.
+    # Raise this when your job has failed, but you don't want to requeue it and try again.
     class RejectJob < StandardError; end
 
+    # Raise thie when you want your job to be executed later (later is defined in {Leveret.configuration.delay_time})
+    class DelayJob < StandardError; end
+
     # Instance methods to mixin with your job
     module InstanceMethods
       # @!attribute params
@@ -59,7 +60,7 @@ module Leveret
 
       # Runs the job and captures any exceptions to turn them into symbols which represent the status of the job
       #
-      # @return [Symbol] :success, :requeue, :reject depending on job success
+      # @return [Symbol] :success, :requeue, :reject, :delay depending on job success
       def run
         Leveret.log.info "Running #{self.class.name} with #{params}"
         perform
@@ -70,6 +71,9 @@ module Leveret
       rescue Leveret::Job::RejectJob
         Leveret.log.warn "Rejecting job #{self.class.name} with #{params}"
         :reject
+      rescue Leveret::Job::DelayJob
+        Leveret.log.warn "Delaying job #{self.class.name} with #{params}"
+        :delay
       rescue StandardError => e
         Leveret.log.error "#{e.message} when processing #{self.class.name} with #{params}"
         Leveret.configuration.error_handler.call(e, self)

data/lib/leveret/message.rb

@@ -0,0 +1,27 @@
+module Leveret
+  # Roll up all of the separate parts of an incoming message into a nice convenient object to move around
+  #
+  # @!attribute delivery_info
+  #   @return [Bunny::DeliveryInfo] Full of useful things like the delivery_tag and routing key
+  # @!attribute properties
+  #   @return [Delivery::Properties] Full of useful things like content-type and priority
+  # @!attribute params
+  #   @return [Parameters] Deserialized params
+  class Message
+    extend Forwardable
+
+    attr_accessor :delivery_info, :properties, :params
+
+    def_delegators :delivery_info, :delivery_tag, :routing_key
+    def_delegators :properties, :priority
+
+    # @param [Bunny::DeliveryInfo] delivery_info Full of useful things like the delivery_tag and routing key
+    # @param [Bunny::Properties] properties Full of useful things like content-type and priority
+    # @param [Parameters] Deserialized params parsed from the message
+    def initialize(delivery_info, properties, params)
+      self.delivery_info = delivery_info
+      self.properties = properties
+      self.params = params
+    end
+  end
+end

data/lib/leveret/queue.rb

@@ -41,7 +41,7 @@ module Leveret
       payload = serialize_payload(payload)
 
       log.debug "Publishing #{payload.inspect} for queue #{name} (Priority: #{priority_id})"
-      queue.publish(payload, persistent: true, routing_key: name, priority: priority_id)
+      exchange.publish(payload, persistent: true, routing_key: name, priority: priority_id)
     end
 
     # Subscribe to this queue and yield a block for every message received. This method does not block, receiving and
@@ -53,16 +53,15 @@ module Leveret
     #
     # @note The receiving block is responsible for acking/rejecting the message. Please see the note for more details.
     #
-    # @yieldparam channel [Bunny::Channel] RabbitMQ channel receiver should use to send ack/reject
-    # @yieldparam delivery_tag [String] The identifier for this message that must be used do ack/reject the message
-    # @yieldparam payload [Parameters] A deserialized version of the payload contained in the message
+    # @yieldparam incoming [Message] Delivery info, properties and the params wrapped up into a convenient object
     #
     # @return [void]
     def subscribe
       log.info "Subscribing to #{name}"
-      queue.subscribe(manual_ack: true) do |delivery_info, _properties, msg|
+      queue.subscribe(manual_ack: true) do |delivery_info, properties, msg|
         log.debug "Received #{msg} from #{name}"
-        yield(queue.channel, delivery_info.delivery_tag, deserialize_payload(msg))
+        incoming = Leveret::Message.new(delivery_info, properties, deserialize_payload(msg))
+        yield incoming
       end
     end
 

data/lib/leveret/result_handler.rb

@@ -0,0 +1,58 @@
+module Leveret
+  # Handles the acknowledgement or rejection of messages after execution
+  class ResultHandler
+    extend Forwardable
+
+    attr_accessor :incoming_message
+
+    def_delegators :Leveret, :log, :delay_queue
+
+    # @param [Message] incoming_message Contains delivery information such as the delivery_tag
+    def initialize(incoming_message)
+      self.incoming_message = incoming_message
+    end
+
+    # Call the appropriate handling method for the result
+    #
+    # @param [Symbol] result Result returned from running the job, one of +:success+, +:reject+ or +:requeue+
+    def handle(result)
+      log.info "[#{delivery_tag}] Job returned #{result}"
+      send(result) if [:success, :reject, :requeue, :delay].include?(result)
+    end
+
+    # Mark the message as acknowledged
+    def success
+      log.debug "[#{delivery_tag}] Acknowledging message"
+      channel.acknowledge(delivery_tag)
+    end
+
+    # Mark the message as rejected (failure)
+    def reject
+      log.debug "[#{delivery_tag}] Rejecting message"
+      channel.reject(delivery_tag)
+    end
+
+    # Reject the message and reinsert it onto it's queue
+    def requeue
+      log.debug "[#{delivery_tag}] Requeueing message"
+      channel.reject(delivery_tag, true)
+    end
+
+    # Acknowledge the message, but publish it onto the delay queue for execution later
+    def delay
+      log.debug ["[#{delivery_tag}] Delaying message"]
+      channel.acknowledge(delivery_tag)
+      delay_queue.republish(incoming_message)
+    end
+
+    private
+
+    def channel
+      incoming_message.delivery_info.channel
+    end
+
+    def delivery_tag
+      incoming_message.delivery_info.delivery_tag
+    end
+  end
+end

data/lib/leveret/version.rb

@@ -1,3 +1,3 @@
 module Leveret
-  VERSION = "0.1.4".freeze
+  VERSION = "0.1.5".freeze
 end

data/lib/leveret/worker.rb

@@ -69,8 +69,8 @@ module Leveret
     # allow us to gracefully cancel these subscriptions when we need to quit.
     def start_subscriptions
       queues.map do |queue|
-        consumers << queue.subscribe do |channel, delivery_tag, payload|
-          fork_and_run(channel, delivery_tag, payload)
+        consumers << queue.subscribe do |incoming_message|
+          fork_and_run(incoming_message)
         end
       end
     end
@@ -88,23 +88,20 @@ module Leveret
     # Detach the main process from the child so we can return to the main loop without waiting for it to finish
     # processing the job.
     #
-    # @param [Bunny::Channel] channel RabbitMQ channel to send the ack message on
-    # @param [String] delivery_tag The identifier that RabbitMQ uses to track the message. This will be used to ack
-    #   or reject the message after processing.
-    # @param [Parameters] payload The job name and parameters the job requires
-    def fork_and_run(channel, delivery_tag, payload)
+    # @param [Message] payload Message meta and payload to process
+    def fork_and_run(incoming_message)
       pid = fork do
         self.process_name = 'leveret-worker-child'
-        log.info "[#{delivery_tag}] Forked to child process #{pid} to run #{payload[:job]}"
+        log.info "[#{incoming_message.delivery_tag}] Forked to child process #{pid} to run #{payload[:job]}"
 
         Leveret.reset_connection!
         Leveret.configuration.after_fork.call
 
-        result = perform_job(payload)
-        log.info "[#{delivery_tag}] Job returned #{result}"
-        ack_message(channel, delivery_tag, result)
+        result = perform_job(incoming_message.params)
+        result_handler = Leveret::ResultHandler.new(incoming_message)
+        result_handler.handle(result)
 
-        log.info "[#{delivery_tag}] Exiting child process #{pid}"
+        log.info "[#{incoming_message.delivery_tag}] Exiting child process #{pid}"
         exit!(0)
       end
 
@@ -120,24 +117,5 @@ module Leveret
       job_klass = Object.const_get(payload[:job])
       job_klass.perform(Leveret::Parameters.new(payload[:params]))
     end
-
-    # Sends a message back to RabbitMQ confirming the completed execution of the message
-    #
-    # @param [Bunny::Channel] channel RabbitMQ channel to send the ack message on
-    # @param [String] delivery_tag The identifier that RabbitMQ uses to track the message. This will be used to ack
-    #   or reject the message after processing.
-    # @param [Symbol] result :success, :reject or :requeue depending on how we want to acknowledge the message
-    def ack_message(channel, delivery_tag, result)
-      if result == :reject
-        log.debug "[#{delivery_tag}] Rejecting message"
-        channel.reject(delivery_tag)
-      elsif result == :requeue
-        log.debug "[#{delivery_tag}] Requeueing message"
-        channel.reject(delivery_tag, true)
-      else
-        log.debug "[#{delivery_tag}] Acknowledging message"
-        channel.acknowledge(delivery_tag)
-      end
-    end
   end
 end

data/lib/leveret.rb

@@ -3,10 +3,13 @@ require 'json'
 require 'logger'
 
 require 'leveret/configuration'
+require 'leveret/delay_queue'
 require 'leveret/job'
 require 'leveret/log_formatter'
+require 'leveret/message'
 require 'leveret/parameters'
 require 'leveret/queue'
+require 'leveret/result_handler'
 require 'leveret/worker'
 require "leveret/version"
 
@@ -54,9 +57,14 @@ module Leveret
       end
     end
 
+    def delay_queue
+      @delay_queue ||= Leveret::DelayQueue.new
+    end
+
     def reset_connection!
       @mq_connection = nil
       @channel = nil
+      @delay_queue = nil
     end
 
     # Logger used throughout Leveret, see {Configuration} for config options.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: leveret
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Dan Wentworth
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-08-01 00:00:00.000000000 Z
+date: 2016-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -101,10 +101,13 @@ files:
 - lib/leveret.rb
 - lib/leveret/cli.rb
 - lib/leveret/configuration.rb
+- lib/leveret/delay_queue.rb
 - lib/leveret/job.rb
 - lib/leveret/log_formatter.rb
+- lib/leveret/message.rb
 - lib/leveret/parameters.rb
 - lib/leveret/queue.rb
+- lib/leveret/result_handler.rb
 - lib/leveret/version.rb
 - lib/leveret/worker.rb
 homepage: https://github.com/darkphnx/leveret