proletariat 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ccf124f3fb011f0dd8f06666daa0f8748c70fe32
+  data.tar.gz: a601e7e559d77d40b30f91be6824d4d51b13af3b
+SHA512:
+  metadata.gz: 22cd707dcdcbfbefbff38a8c4c4bf74cceefa87c42c30706f9028a89d2039fc0095ac1deff157f5efa9c4e60c881e4fe6cd2b8f10596142ef257cfbb54460cdb
+  data.tar.gz: 0160aced5ba612299ddb023b8cf19c361a2d02e9212bfb8c2e1d79dfbe2db07d017378ba7b9b029d894e7d18903ee371ec34623cef2941e611e4f80148a34fe6

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.ruby-version
+.ruby-gemset

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,8 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in proletariat.gemspec
+gemspec
+
+platforms :rbx do
+  gem 'rubysl'
+end

data/README.md

@@ -0,0 +1,93 @@
+# Proletariat: Background Workers Unite!
+
+Lightweight background processing in Ruby powered by RabbitMQ and the excellent concurrent-ruby gem.
+
+### Warning!
+
+This software is early-alpha, may contain bugs and change considerably in the near future.
+
+For production use I recommend the better supported and more fully-featured [Sneakers gem](https://github.com/jondot/sneakers).
+
+## Installation
+
+Add this line to your application's `Gemfile`:
+
+    gem 'proletariat'
+
+And run:
+
+    $ bundle
+
+## How to use
+
+### RabbitMQ connection config
+
+If you aren't using default RabbitMQ connection settings, ensure the `RABBITMQ_URL` env variable is present. Here's how that might look in your `.env` if you use Foreman:
+
+	RABBITMQ_URL=amqp://someuser:somepass@127.0.0.1/another_vhost
+
+### Setting up a Worker
+
+Your worker classes should inherit from `Proletariat::Worker` and implement the `#work` method.
+
+Proletariat works exclusively on RabbitMQ Topic exchanges and routing keys can be bound via a call to `.listen_on`. This can be called multiple times to bind to multiple keys.
+
+The `#work` method should return `:ok` on success or `:drop` / `:requeue` on failure.
+
+Here's a complete example:
+	
+	class SendUserIntroductoryEmail < Proletariat::Worker
+	  listen_on 'user.created'
+	
+	  def work(message)
+	    params = JSON.parse(message)
+	
+	    UserMailer.introductory_email(params).deliver!
+	
+	    publish 'email_sent.user.introductory', {id: params['id']}.to_json
+	
+	    :ok
+	  end
+	end
+	
+### Select your Workers
+
+If you are using Rails just create a `proletariat.rb` file in your initializers directory.
+
+	Proletariat.configure worker_classes: [SendUserIntroductoryEmail, SomeOtherWorker]
+	Proletariat.run!
+	
+Or define the `WORKERS` env variable.
+
+	WORKERS=SendUserIntroductoryEmail,SomeOtherWorker
+	
+### Testing with Cucumber
+
+Add the following to your `env.rb`:
+
+	require 'proletariat/cucumber'
+	
+Use the provided helpers in your step definitions to synchronize your test suite with your workers without sacrificing the ability to test in production-like environment:
+
+	When(/^I submit a valid 'register user' form$/) do
+	  wait_for message.on_topic('email_sent.user.introductory') do
+	    visit   ...
+	    fill_in ...
+	    submit  ...
+	  end
+	end
+	
+	Then(/^the user should receive an introductory email$/) do
+	  expect(unread_emails_for(new_user_email).size).to eq 1
+	end
+
+## FAQ
+
+#### Why build another RabbitMQ background worker library?
+
+I wanted a library which shared one RabbitMQ connection across all of the workers on a given process. Many hosted RabbitMQ platforms tightly limit the max number of connections.
+
+## TODO
+- Improve test suite :(
+- Add command line interface
+- Abstract retry strategies

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/proletariat/concerns/logging.rb

@@ -0,0 +1,17 @@
+module Proletariat
+  module Concerns
+    # Public: Mixin to handle logging concerns.
+    module Logging
+      # Public: Logs info to the process-wide logger.
+      #
+      # message - The message to be logged.
+      #
+      # Returns nil.
+      def log_info(message)
+        Proletariat.logger.info "#{self.class.name} #{object_id}: #{message}"
+
+        nil
+      end
+    end
+  end
+end

data/lib/proletariat/cucumber.rb

@@ -0,0 +1,20 @@
+require 'proletariat/testing'
+
+# Mix testing helpers into World.
+World(Proletariat::Testing)
+
+# Hide logs by default.
+AfterConfiguration do |config|
+  Proletariat.logger = Logger.new('/dev/null')
+end
+
+# Ensure Proletariat running before each test.
+Before do
+  Proletariat.run! unless Proletariat.running?
+end
+
+# Stop workers and purge queues between scenarios.
+After do
+  Proletariat.stop
+  Proletariat.purge
+end

data/lib/proletariat/manager.rb

@@ -0,0 +1,99 @@
+module Proletariat
+  # Public: Maintains a pool of worker threads and a RabbitMQ subscriber
+  #         thread. Uses information from the worker class to generate queue
+  #         config.
+  class Manager < Concurrent::Supervisor
+    # Public: Creates a new Manager instance.
+    #
+    # connection    - An open Bunny::Session object.
+    # exchange_name - A String of the RabbitMQ topic exchange.
+    # worker_class  - A subclass of Proletariat::Worker to handle messages.
+    # options       - A Hash of additional optional parameters (default: {}):
+    #                 :worker_threads - The size of the worker thread pool.
+    def initialize(connection, exchange_name, worker_class, options = {})
+      super()
+
+      @connection     = connection
+      @exchange_name  = exchange_name
+      @worker_class   = worker_class
+      @worker_threads = options.fetch :worker_threads, 3
+
+      create_worker_pool
+      create_subscriber
+
+      worker_pool.each { |worker| add_worker worker }
+      add_worker subscriber
+    end
+
+    # Public: Purge the RabbitMQ queue.
+    #
+    # Returns nil.
+    def purge
+      subscriber.purge
+
+      nil
+    end
+
+    private
+
+    # Internal: Returns an open Bunny::Session object.
+    attr_reader :connection
+
+    # Internal: Returns the name of the RabbitMQ topic exchange.
+    attr_reader :exchange_name
+
+    # Internal: Returns the Subscriber actor for this Manager.
+    attr_reader :subscriber
+
+    # Internal: Returns the subclass of Worker used to process messages.
+    attr_reader :worker_class
+
+    # Internal: Returns the pool of initialized workers.
+    attr_reader :worker_pool
+
+    # Internal: Returns a shared mailbox for the pool of workers.
+    attr_reader :workers_mailbox
+
+    # Internal: Returns the number of worker threads in the worker pool.
+    attr_reader :worker_threads
+
+    # Internal: Assign a new Subscriber instance (configured for the current
+    #           worker type) to the manager's subscriber property.
+    #
+    # Returns nil.
+    def create_subscriber
+      @subscriber = Subscriber.new(
+        connection,
+        workers_mailbox,
+        generate_queue_config
+      )
+
+      nil
+    end
+
+    # Internal: Assign new Concurrent::Poolbox and Array[Worker] to the
+    #           manager's workers_mailbox and worker_pool properties
+    #           respectively.
+    #
+    # Returns nil.
+    def create_worker_pool
+      @workers_mailbox, @worker_pool = worker_class.pool(worker_threads)
+
+      nil
+    end
+
+    # Internal: Builds a new QueueConfig object passing in some settings from
+    #           worker_class.
+    #
+    # Returns a new QueueConfig instance.
+    def generate_queue_config
+      QueueConfig.new(
+        worker_class.name,
+        exchange_name,
+        worker_class.routing_keys,
+        worker_threads,
+        false
+      )
+    end
+  end
+end

data/lib/proletariat/publisher.rb

@@ -0,0 +1,87 @@
+module Proletariat
+  # Public: Listens for messages in it's mailbox and publishes
+  #         these to a RabbitMQ topic exchange.
+  class Publisher < Concurrent::Actor
+    include Concerns::Logging
+
+    # Public: Creates a new Publisher instance.
+    #
+    # connection    - An open Bunny::Session object.
+    # exchange_name - A String of the RabbitMQ topic exchange.
+    def initialize(connection, exchange_name)
+      @channel  = connection.create_channel
+      @exchange = channel.topic(exchange_name, durable: true)
+    end
+
+    # Public: Called by the Concurrent framework to handle new mailbox
+    #         messages. Overridden in this subclass to push messages to a
+    #         RabbitMQ topic exchange.
+    #
+    # to      - The routing key for the message to as a String. In accordance
+    #           with the RabbitMQ convention you can use the '*' character to
+    #           replace one word and the '#' to replace many words.
+    # message - The message as a String.
+    #
+    # Returns nil.
+    def act(to, message)
+      publish(to, message)
+
+      nil
+    end
+
+    # Public: Called by the Concurrent framework on actor start. Overridden in
+    #         this subclass to log the status of the publisher.
+    #
+    # Returns nil.
+    def on_run
+      super
+      log_info 'Now online'
+
+      nil
+    end
+
+    # Public: Called by the Concurrent framework on actor stop. Overridden in
+    #         this subclass to log the status of the publisher.
+    def on_stop
+      log_info 'Attempting graceful shutdown.'
+      wait_for_publish_queue unless queue.empty?
+
+      super
+
+      log_info 'Now offline'
+
+      nil
+    end
+
+    private
+
+    # Internal: Returns the Bunny::Channel in use.
+    attr_reader :channel
+
+    # Internal: Returns the Bunny::Exchange in use.
+    attr_reader :exchange
+
+    # Internal: Handles the actual message send to the exchange.
+    #
+    # to      - The routing key.
+    # message - The message as a String.
+    #
+    # Returns nil.
+    def publish(to, message)
+      exchange.publish message, routing_key: to, persistent: true
+
+      nil
+    end
+
+    # Internal: Blocks until each message has been published.
+    #
+    # Returns nil.
+    def wait_for_publish_queue
+      log_info 'Waiting for work queue to drain.'
+
+      publish(*queue.pop.message) until queue.empty?
+
+      nil
+    end
+  end
+end

data/lib/proletariat/queue_config.rb

@@ -0,0 +1,26 @@
+# Internal: Value object to hold RabbitMQ settings.
+class QueueConfig < Struct.new(:worker_name,
+                               :exchange_name,
+                               :routing_keys,
+                               :prefetch,
+                               :auto_delete)
+  # Public: Create an underscored RabbitMQ queue name from the worker_name.
+  #
+  # Examples
+  #
+  #   config = QueueConfig.new('ExampleWorker', ...)
+  #   config.queue_name
+  #   # => 'example_worker'
+  #
+  # Returns the queue name as a String.
+  def queue_name
+    @queue_name ||= begin
+      worker_name
+        .gsub(/::/, '/')
+        .gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+        .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+        .tr('-', '_')
+        .downcase
+    end
+  end
+end

data/lib/proletariat/runner.rb

@@ -0,0 +1,149 @@
+module Proletariat
+  # Public: Sets up a supervisor which maintains a single Publisher and a
+  # per-worker Manager instance.
+  class Runner
+    extend Forwardable
+
+    # Public: Delegate lifecycle calls to the supervisor.
+    def_delegators :supervisor, :run, :run!, :stop, :running?
+
+    # Public: Returns an open Bunny::Session object.
+    attr_reader :connection
+
+    # Public: Returns the name of the RabbitMQ topic exchange.
+    attr_reader :exchange_name
+
+    # Public: Creates a new Runner instance. Default options should be fine for
+    #         most scenarios.
+    #
+    # options - A Hash of options (default: {}):
+    #             :connection        - An open RabbitMQ::Session object.
+    #             :exchange_name     - The RabbitMQ topic exchange name as a
+    #                                  String.
+    #             :publisher_threads - The size of the publisher thread pool.
+    #             :supervisor        - A Supervisor instance.
+    #             :worker_classes    - An Array of Worker subclasses.
+    #             :worker_threads    - The size of the worker thread pool.
+    def initialize(options = {})
+      @connection        = options.fetch :connection, create_connection
+      @exchange_name     = options.fetch :exchange_name, DEFAULT_EXCHANGE_NAME
+      @publisher_threads = options.fetch :publisher_threads, 2
+      @supervisor        = options.fetch :supervisor, create_supervisor
+      @worker_classes    = options.fetch :worker_classes, []
+      @worker_threads    = options.fetch :worker_threads, 3
+
+      @managers = []
+
+      create_publisher_pool
+
+      add_publishers_to_supervisor
+      add_workers_to_supervisor
+    end
+
+    # Public: Publishes a message to RabbitMQ via the publisher pool.
+    #
+    # to      - The routing key for the message to as a String. In accordance
+    #           with the RabbitMQ convention you can use the '*' character to
+    #           replace one word and the '#' to replace many words.
+    # message - The message as a String.
+    #
+    # Returns nil.
+    def publish(to, message)
+      publishers_mailbox.post to, message
+
+      nil
+    end
+
+    # Public: Purge the RabbitMQ queues.
+    #
+    # Returns nil.
+    def purge
+      managers.each { |manager| manager.purge }
+
+      nil
+    end
+
+    private
+
+    # Internal: Returns an Array of the currently supervised Managers.
+    attr_reader :managers
+
+    # Internal: Returns the pool of initialized publishers.
+    attr_reader :publisher_pool
+
+    # Internal: Returns a shared mailbox for the pool of publishers.
+    attr_reader :publishers_mailbox
+
+    # Internal: Returns the number of publisher threads in the publisher pool.
+    attr_reader :publisher_threads
+
+    # Internal: Returns the supervisor instance.
+    attr_reader :supervisor
+
+    # Internal: Returns an Array of Worker subclasses.
+    attr_reader :worker_classes
+
+    # Internal: Returns the number of worker threads per manager.
+    attr_reader :worker_threads
+
+    # Internal: Adds each publisher in the publisher_pool to the supervisor.
+    #
+    # Returns nil.
+    def add_publishers_to_supervisor
+      publisher_pool.each { |publisher| supervisor.add_worker publisher }
+
+      nil
+    end
+
+    # Internal: Creates a Manager per worker_class and adds these to the
+    #           supervisor.
+    #
+    # Returns nil.
+    def add_workers_to_supervisor
+      worker_classes.each do |worker_class|
+        manager = create_manager(worker_class)
+        @managers << manager
+        supervisor.add_worker manager
+      end
+
+      nil
+    end
+
+    # Internal: Creates a new Bunny::Session and opens it.
+    #
+    # Returns an open Bunny::Session instance.
+    def create_connection
+      new_connection = Bunny.new
+      new_connection.start
+
+      new_connection
+    end
+
+    # Internal: Assign new Concurrent::Poolbox and Array[Publisher] to the
+    #           manager's publishers_mailbox and publisher_pool properties
+    #           respectively.
+    #
+    # Returns nil.
+    def create_publisher_pool
+      @publishers_mailbox, @publisher_pool = Publisher.pool(publisher_threads,
+                                                            connection,
+                                                            exchange_name)
+    end
+
+    # Internal: Creates a new Concurrent::Supervisor.
+    #
+    # Returns a Concurrent::Supervisor instance.
+    def create_supervisor
+      Concurrent::Supervisor.new
+    end
+
+    # Internal: Creates a Manager for a given Worker subclass adding relevant
+    #           arguments from Runner properties.
+    #
+    # Returns a Manager instance.
+    def create_manager(worker_class)
+      Manager.new(connection, exchange_name, worker_class,
+                  worker_threads: worker_threads)
+    end
+  end
+end