proletariat 0.0.6 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b5b2573ecbcb0bba6ed669afbe997c428fe5892e
-  data.tar.gz: f04b0420cf1d8dbf6f142a63a7441d14627120db
+  metadata.gz: 6cb914420e4992314911f3e2fc3a69cfdaf17cc1
+  data.tar.gz: fbc67bb27579bb70284d67b38047dae9ffa5ba72
 SHA512:
-  metadata.gz: 4d3813e197e25da9ebd15b85cfcfd114d77c879dc42da712534c75e394e4c9f55781cdacbe8e7284fb0f2db0df7c24f5e0ffc96bd76e22835bf30df8756f2f5c
-  data.tar.gz: 1b0fe700ce9145963d68c28adead0eda8c36b4ee6f50d60cb9f04faa7a2e91e6164281259d48f33f5fc7f4254ce016f6337b80d20d7d9c6d767d4e83d8dd46f8
+  metadata.gz: a71faf28c2fceec5f5f9bdb5cef95732bb3fae7fb5f43738155300b44b83884790ffa6096bff75cf7196b284b0f9aafc0bdea14deffd3874229fc17fc74f8d5c
+  data.tar.gz: acce71270fc40d16ce387ae11512566345d1aedd5412219358f9308c1940ebe6af9841b865bbdf2ee4c5c6b821521fb73dd1d8c5e227117cbe2824463b2a5ec0

data/Gemfile

@@ -3,6 +3,8 @@ source "http://rubygems.org"
 # Specify your gem's dependencies in proletariat.gemspec
 gemspec
 
+gem 'concurrent-ruby', github: 'ruby-concurrency/concurrent-ruby'
+
 platforms :rbx do
   gem 'rubysl'
 end

data/README.md

@@ -12,9 +12,10 @@ For production use I recommend the better supported and more fully-featured [Sne
 
 ## Installation
 
-Add this line to your application's `Gemfile`:
+Add these lines to your application's `Gemfile`:
 
     gem 'proletariat'
+    gem 'concurrent-ruby', github: 'ruby-concurrency/concurrent-ruby'
 
 And run:
 

data/lib/proletariat.rb

@@ -6,18 +6,23 @@ require 'logger'
 require 'forwardable'
 
 require 'proletariat/concurrency/actor'
-require 'proletariat/concurrency/supervisor'
+require 'proletariat/concurrency/poolable_actor'
 
 require 'proletariat/util/worker_description_parser'
 
 require 'proletariat/configuration'
+require 'proletariat/exception_handler'
 require 'proletariat/manager'
+require 'proletariat/message'
 require 'proletariat/publisher'
 require 'proletariat/queue_config'
 require 'proletariat/runner'
 require 'proletariat/subscriber'
 require 'proletariat/worker'
 
+require 'proletariat/exception_handler/drop'
+require 'proletariat/exception_handler/exponential_backoff'
+
 # Public: Creates the Proletariat namespace and holds a process-wide Runner
 #         instance as well as access to the configuration attributes.
 module Proletariat
@@ -25,7 +30,7 @@ module Proletariat
     extend Forwardable
 
     # Public: Delegate lifecycle calls to the process-wide Runner.
-    def_delegators :runner, :run, :run!, :stop, :running?, :publish, :purge
+    def_delegators :runner, :run, :running?, :stop
 
     # Public: Allows configuration of Proletariat via given block.
     #
@@ -38,6 +43,10 @@ module Proletariat
       nil
     end
 
+    def publish(to, body = '', headers = {})
+      publisher_pool << Message.new(to, body, headers)
+    end
+
     def runner
       @runner ||= Runner.new
     end
@@ -56,5 +65,9 @@ module Proletariat
     def config
       @config ||= Configuration.new
     end
+
+    def publisher_pool
+      @publisher_pool ||= Publisher.pool(Proletariat.publisher_threads)
+    end
   end
 end

data/lib/proletariat/concurrency/actor.rb

@@ -1,71 +1,16 @@
-module Proletariat
-  # Public: Interface abstraction for Concurrent::Actor. Creates a delegate
-  #         instance from given class and arguments and delegates events to it.
-  class Actor < Concurrent::Actor
-    # Public: Creates a new Actor instance.
-    #
-    # delegate_class - The class to instantiate as a delegate.
-    # *arguments     - The arguments to pass to delegate_class.new
-    def initialize(delegate_class, *arguments)
-      @delegate = delegate_class.new(*arguments)
-    end
-
-    # Internal: Called by the Concurrent framework to handle new mailbox
-    #           messages. Overridden in this subclass to call the #work method
-    #           with the given arguments on the delegate.
-    #
-    # *arguments - The arguments to pass to delegate#work
-    #
-    # Returns nil.
-    def act(*arguments)
-      delegate.work(*arguments)
-    end
-
-    # Internal: Called by the Concurrent framework on actor start. Overridden
-    #           in this subclass to call the #starting and #started methods on
-    #           the delegate.
-    #
-    # Returns nil.
-    def on_run
-      delegate.starting if delegate.respond_to?(:starting)
-
-      super
-
-      delegate.started if delegate.respond_to?(:started)
-
-      nil
-    end
-
-    # Internal: Called by the Concurrent framework on actor start. Overridden
-    #           in this subclass to call the #stopping and #stopped methods on
-    #           the delegate. Ensures queue is drained before calling #stopped.
-    #
-    # Returns nil.
-    def on_stop
-      delegate.stopping if delegate.respond_to?(:stopping)
+require 'proletariat/concurrency/actor_common'
 
-      wait_for_queue_to_drain unless queue.empty?
-
-      super
-
-      delegate.stopped if delegate.respond_to?(:stopped)
-
-      nil
-    end
-
-    private
-
-    # Internal: Returns the delegate instance.
-    attr_reader :delegate
-
-    # Internal: Blocks until each queued message has been handled by the
-    #           delegate #work method.
-    #
-    # Returns nil.
-    def wait_for_queue_to_drain
-      delegate.work(*queue.pop.message) until queue.empty?
-
-      nil
+module Proletariat
+  # Public: Interface abstraction for Concurrent::Actor.
+  class Actor < Concurrent::Actor::RestartingContext
+    include ActorCommon
+
+    def on_message(message)
+      if respond_to?(:work_method)
+        send work_method, message
+      else
+        work message
+      end
     end
   end
 end

data/lib/proletariat/concurrency/actor_common.rb

@@ -0,0 +1,30 @@
+module Proletariat
+  # Internal: Common behavior for actor base classes.
+  module ActorCommon
+    def self.included(base)
+      base.class_exec do
+        def initialize(*args)
+          starting if respond_to?(:starting)
+
+          super
+
+          started if respond_to?(:started)
+        end
+      end
+    end
+
+    def on_event(event)
+      if event == :terminated
+        stopping if respond_to?(:stopping)
+
+        cleanup if respond_to?(:cleanup)
+
+        super
+
+        stopped if respond_to?(:stopped)
+      else
+        super
+      end
+    end
+  end
+end

data/lib/proletariat/concurrency/poolable_actor.rb

@@ -0,0 +1,25 @@
+require 'proletariat/concurrency/actor_common'
+
+module Proletariat
+  # Public: Interface abstraction for a pool of Concurrent::Actor instances.
+  class PoolableActor < Concurrent::Actor::Utils::AbstractWorker
+    include ActorCommon
+
+    def on_message(message)
+      if respond_to?(:work_method)
+        send work_method, message
+      else
+        work message
+      end
+    ensure
+      @balancer << :subscribe
+    end
+
+    def self.pool(pool_size, suffix = '')
+      Concurrent::Actor::Utils::
+        Pool.spawn!("#{to_s}_pool", pool_size) do |b, i|
+          spawn(name: "#{to_s}_#{i}_#{suffix}", supervise: true, args: [b])
+        end
+    end
+  end
+end

data/lib/proletariat/configuration.rb

@@ -87,6 +87,21 @@ module Proletariat
       @publisher_threads ||= DEFAULT_PUBLISHER_THREADS
     end
 
+    # Public: Enables test mode. Queues will auto-delete after use.
+    #
+    # Returns nil.
+    def test_mode!
+      @test_mode = true
+    end
+
+    # Public: Returns whether test mode is enabled or not.
+    #
+    # Returns true if test mode enabled.
+    # Returns false if test mode disabled.
+    def test_mode?
+      !!@test_mode
+    end
+
     # Public: Returns the set worker classes or a default pulled from the
     #         WORKERS env variable.
     #

data/lib/proletariat/cucumber.rb

@@ -7,18 +7,16 @@ World(Proletariat::Testing)
 AfterConfiguration do |_|
   Proletariat.configure do
     config.logger = Logger.new('/dev/null')
+    config.test_mode!
   end
-
-  Proletariat.purge
 end
 
 # Ensure Proletariat running before each test.
 Before do
-  Proletariat.run! unless Proletariat.running?
+  Proletariat.run unless Proletariat.running?
 end
 
 # Stop workers and purge queues between scenarios.
 After do
   Proletariat.stop
-  Proletariat.purge
 end

data/lib/proletariat/exception_handler.rb

@@ -0,0 +1,45 @@
+require 'proletariat/concerns/logging'
+
+module Proletariat
+  # Public: Handles messages whose work raised an exception. Should overwrite
+  #         the #work method to implement retry/drop logic.
+  class ExceptionHandler < Actor
+    include Concerns::Logging
+
+    def initialize(queue_name)
+      @queue_name = queue_name
+      setup
+    end
+
+    # Internal: Handles the Actor mailbox. Delegates work to #work.
+    #
+    # message - A Message to send.
+    def actor_work(message)
+      work message.body, message.to, message.headers if message.is_a?(Message)
+    end
+
+    # Public: Callback hook for initialization.
+    def setup
+    end
+
+    # Public: Handles an incoming message to perform background work.
+    #
+    # body        - The failed message body.
+    # to          - The failed message's routing key.
+    # headers     - The failed message's headers.
+    #
+    # Raises NotImplementedError unless implemented in subclass.
+    def work(body, to, headers)
+      fail NotImplementedError
+    end
+
+    # Public: Use #actor_work to handle the actor mailbox.
+    def work_method
+      :actor_work
+    end
+
+    protected
+
+    attr_reader :queue_name
+  end
+end

data/lib/proletariat/exception_handler/drop.rb

@@ -0,0 +1,15 @@
+module Proletariat
+  # Internal: Exception handler which just drops failed messages.
+  class Drop < ExceptionHandler
+    # Public: Does nothing with the failed messages.
+    #
+    # body        - The failed message body.
+    # to          - The failed message's routing key.
+    # headers     - The failed message's headers.
+    #
+    # Returns nil.
+    def work(message, to, headers)
+      nil
+    end
+  end
+end

data/lib/proletariat/exception_handler/exponential_backoff.rb

@@ -0,0 +1,130 @@
+module Proletariat
+  # Internal: Exception handler with an exponential back-off strategy. Uses
+  #           dead-letter queues.
+  class ExponentialBackoff < ExceptionHandler
+    # Public: Called on actor termination. Used to stop consumption off the
+    #         requeue queue and close the channel.
+    #
+    # Returns nil.
+    def cleanup
+      @consumer.cancel if @consumer
+      @channel.close if @channel && channel.open?
+    end
+
+    # Public: Callback hook for initialization. Kicks off the queue setup.
+    def setup
+      setup_requeue_queue
+      consume_requeue
+      setup_retry_queues
+    end
+
+    # Public: Puts messages into a delay queue to be retried in the future.
+    #
+    # body        - The failed message body.
+    # to          - The failed message's routing key.
+    # headers     - The failed message's headers.
+    #
+    # Returns nil.
+    def work(message, to, headers)
+      failures = queue_for_x_death(headers['x-death'])
+
+      exchange.publish(message,
+                       routing_key: "#{queue_name}_delay_#{failures}",
+                       persistent: !Proletariat.test_mode?,
+                       headers: headers.merge('proletariat-to' => to))
+
+      nil
+    end
+
+    private
+
+    # Internal: Returns the Bunny::Channel in use.
+    def channel
+      @channel ||= Proletariat.connection.create_channel
+    end
+
+    # Internal: Starts a consumer on the requeue queue which puts messages back
+    #           onto the main queue.
+    #
+    # Returns nil.
+    def consume_requeue
+      @consumer = @requeue.subscribe do |info, props, body|
+        Proletariat.publish(props.headers['proletariat-to'], body,
+                            props.headers)
+
+        nil
+      end
+    end
+
+    # Internal: Returns the Bunny::Exchange in use.
+    def exchange
+      @exchange ||= channel.direct(exchange_name,
+                                   durable: !Proletariat.test_mode?,
+                                   auto_delete: Proletariat.test_mode?)
+    end
+
+    # Internal: Returns a new exchange name for a direct exchange.
+    def exchange_name
+      "#{Proletariat.exchange_name}_retry"
+    end
+
+    # Internal: Determines which delay queue a failed message should go in
+    #           based on number of past fails shown in x-death header.
+    #
+    # header - the x-death header.
+    #
+    # Returns an Integer.
+    def queue_for_x_death(header)
+      if header
+        [header.length, retry_delay_times.length - 1].min
+      else
+        0
+      end
+    end
+
+    # Internal: Calculates an exponential retry delay based on the previous
+    #           number of failures. Capped with configuration setting.
+    #
+    # Returns the delay in seconds as a Fixnum.
+    def retry_delay_times
+      @delay_times ||= begin
+        (1..Float::INFINITY)
+          .lazy
+          .map { |i| i**i }
+          .take_while { |i| i < Proletariat.max_retry_delay }
+          .to_a
+          .push(Proletariat.max_retry_delay)
+          .map { |seconds| seconds * 1000 }
+      end
+    end
+
+    # Internal: Creates the requeue queue and binds it to the exchange.
+    def setup_requeue_queue
+      @requeue = channel.queue("#{queue_name}_requeue",
+                               durable: !Proletariat.test_mode?,
+                               auto_delete: Proletariat.test_mode?)
+
+      @requeue.bind(exchange, routing_key: "#{queue_name}_requeue")
+    end
+
+    # Internal: Create a delay queue and binds it to the exchange.
+    def setup_retry_queue(delay, index)
+      channel.queue("#{queue_name}_delay_#{index}",
+                    durable: !Proletariat.test_mode?,
+                    auto_delete: Proletariat.test_mode?,
+                    arguments: {
+                      'x-dead-letter-exchange' => exchange_name,
+                      'x-dead-letter-routing-key' => "#{queue_name}_requeue",
+                      'x-message-ttl' => delay
+                    }
+      ).bind(exchange, routing_key: "#{queue_name}_delay_#{index}")
+    end
+
+    # Internal: Creates the delay queues based on the max retry time.
+    def setup_retry_queues
+      retry_delay_times.each_with_index.map do |delay, index|
+        setup_retry_queue(delay, index)
+      end
+    end
+  end
+end