toiler 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1411df909332ded188599a79d720d618e46bf5bd
+  data.tar.gz: 1e665ab6d968eaa0a85955202e5d78e8fedd99a0
+SHA512:
+  metadata.gz: b2fa690e79c8a88c53ef0e4df2445f57f1eb5212db96b3bb61904d035eed5bcefef868fdf37c949fe650e70875b210acfbcfe836bd7c2fdf2d1a3accf14bc96f
+  data.tar.gz: bf9f7db7c915ed650d81597ac0c0eb74e0a65215fdad13e0c82d7a843ff9d5d76ab4d7978882b61a01362237c2b358c8cbc7643534f7c1e0a1c719997d3141fc

data/.gitignore

@@ -0,0 +1,35 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem 'celluloid'
+gem 'celluloid-io'
+gem 'aws-sdk'

data/Gemfile.lock

@@ -0,0 +1,32 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    aws-sdk (2.0.41)
+      aws-sdk-resources (= 2.0.41)
+    aws-sdk-core (2.0.41)
+      builder (~> 3.0)
+      jmespath (~> 1.0)
+      multi_json (~> 1.0)
+    aws-sdk-resources (2.0.41)
+      aws-sdk-core (= 2.0.41)
+    builder (3.2.2)
+    celluloid (0.16.0)
+      timers (~> 4.0.0)
+    celluloid-io (0.16.2)
+      celluloid (>= 0.16.0)
+      nio4r (>= 1.1.0)
+    hitimes (1.2.2)
+    jmespath (1.0.2)
+      multi_json (~> 1.0)
+    multi_json (1.11.0)
+    nio4r (1.1.0)
+    timers (4.0.1)
+      hitimes
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  aws-sdk
+  celluloid
+  celluloid-io

data/LICENSE

@@ -0,0 +1,6 @@
+Copyright (c) Sebastian Schepens
+
+Toiler is an Open Source project licensed under the terms of
+the LGPLv3 license.  Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
+for license text.
+

data/README.md

@@ -0,0 +1,110 @@
+##Toiler
+Toiler is a AWS SQS long-polling thread-based message processor.
+It's based on [shoryuken](https://github.com/phstc/shoryuken) but takes
+a different approach at loadbalancing and uses long-polling.
+
+##Features
+###Concurrency
+Toiler allows to specify the amount of processors (threads) that should be spawned for each queue.
+Instead of [shoryuken's](https://github.com/phstc/shoryuken) loadbalancing  approach, Toiler delegates this work to the kernel scheduling threads.
+
+###Long-Polling
+A Fetcher thread is spawned for each queue.
+Fetchers are resposible for polling SQS and retreiving messages.
+They are optimised to not bring more messages than the amount of processors avaiable for such queue.
+By long-polling fetchers wait for a configurable amount of time for messages to become available on a single request, this prevents unneccesarilly requesting messages when there are none.
+
+###Message Parsing
+Workers can configure a parser Class or Proc to parse an SQS message body before being processed.
+
+###Batches
+Toiler allows a Worker to be able to receive a batch of messages instead of a single one.
+
+##Instalation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'toiler'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install toiler
+
+## Usage
+
+### Worker class
+
+```ruby
+class MyWorker
+  include Toiler::Worker
+
+  toiler_options queue: 'default', concurrency: 5, auto_delete: true
+  toiler_options parser: :json
+
+  # toiler_options parser: ->(sqs_msg){ REXML::Document.new(sqs_msg.body) }
+  # toiler_options parser: MultiJson
+  # toiler_options auto_visibility_timeout: true
+  # toiler_options batch: true
+
+  def perform(sqs_msg, body)
+    puts body
+  end
+end
+```
+
+### Configuration
+
+```yaml
+aws:
+  access_key_id:      ...       # or <%= ENV['AWS_ACCESS_KEY_ID'] %>
+  secret_access_key:  ...       # or <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
+  region:             us-east-1 # or <%= ENV['AWS_REGION'] %>
+wait: 20                        # The time in seconds to wait for messages during long-polling
+```
+
+### Rails Integration
+
+You can tell Toiler to load your Rails application by passing the `-R` or `--rails` flag to the "toiler" command.
+
+If you load Rails, and assuming your workers are located in the `app/workers` directory, they will be auto-loaded. This means you don't need to require them explicitly with `-r`.
+
+
+### Start Toiler
+
+```shell
+bundle exec toiler -r worker.rb -C toiler.yml
+```
+
+Other options:
+
+```bash
+toiler --help
+
+    -d, --daemon                     Daemonize process
+    -r, --require [PATH|DIR]         Location of the worker
+    -C, --config PATH                Path to YAML config file
+    -R, --rails                      Load Rails
+    -L, --logfile PATH               Path to writable logfile
+    -P, --pidfile PATH               Path to pidfile
+    -v, --verbose                    Print more verbose output
+    -h, --help                       Show help
+```
+
+
+## Credits
+
+Much of the credit goes to [Pablo Cantero](https://github.com/phstc), creator of [Shoryuken](https://github.com/phstc/shoryuken), and [everybody who contributed to it](https://github.com/phstc/shoryuken/graphs/contributors).
+
+## Contributing
+
+1. Fork it ( https://github.com/sschepens/toiler/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require 'bundler/gem_tasks'

data/bin/toiler

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require 'toiler'
+
+begin
+  Toiler::CLI.instance.run(ARGV)
+rescue => e
+  raise e if $DEBUG
+  STDERR.puts e.message
+  STDERR.puts e.backtrace.join("\n")
+  exit 1
+end

data/lib/toiler.rb

@@ -0,0 +1,80 @@
+require 'aws-sdk'
+require 'toiler/core_ext'
+require 'toiler/message'
+require 'toiler/queue'
+require 'toiler/worker'
+require 'toiler/environment_loader'
+require 'toiler/logging'
+require 'toiler/cli'
+require 'toiler/version'
+
+module Toiler
+  @worker_registry = {}
+  @worker_class_registry = {}
+  @options = {
+    aws: {}
+  }
+
+  module_function
+
+  def options
+    @options
+  end
+
+  def logger
+    Toiler::Logging.logger
+  end
+
+  def worker_class_registry
+    @worker_class_registry
+  end
+
+  def worker_registry
+    @worker_registry
+  end
+
+  def queues
+    @worker_registry.keys
+  end
+
+  def fetcher(queue)
+    Celluloid::Actor["fetcher_#{queue}".to_sym]
+  end
+
+  def set_fetcher(queue, val)
+    Celluloid::Actor["fetcher_#{queue}".to_sym] = val
+  end
+
+  def processor_pool(queue)
+    Celluloid::Actor["processor_pool_#{queue}".to_sym]
+  end
+
+  def set_processor_pool(queue, val)
+    Celluloid::Actor["processor_pool_#{queue}".to_sym] = val
+  end
+
+  def manager
+    Celluloid::Actor[:manager]
+  end
+
+  def set_manager(val)
+    Celluloid::Actor[:manager] = val
+  end
+
+  def timer
+    Celluloid::Actor[:timer]
+  end
+
+  def set_timer(val)
+    Celluloid::Actor[:timer] = val
+  end
+
+  def default_options
+    {
+      auto_visibility_timeout: false,
+      concurrency: 1,
+      auto_delete: false,
+      batch: false
+    }
+  end
+end

data/lib/toiler/cli.rb

@@ -0,0 +1,141 @@
+require 'singleton'
+require 'optparse'
+require 'toiler'
+
+module Toiler
+  # See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
+  class Shutdown < Interrupt; end
+
+  class CLI
+    include Singleton
+
+    def run(args)
+      self_read, self_write = IO.pipe
+
+      %w(INT TERM USR1 USR2 TTIN).each do |sig|
+        begin
+          trap sig do
+            self_write.puts(sig)
+          end
+        rescue ArgumentError
+          puts "Signal #{sig} not supported"
+        end
+      end
+
+      options = parse_cli_args(args)
+
+      EnvironmentLoader.load(options)
+      daemonize
+      write_pid
+      load_celluloid
+
+      begin
+        @supervisor = Supervisor.new
+
+        while (readable_io = IO.select([self_read]))
+          signal = readable_io.first[0].gets.strip
+          handle_signal(signal)
+        end
+      rescue Interrupt
+        @supervisor.stop
+        exit 0
+      end
+    end
+
+    private
+
+    def handle_signal(_signal)
+      fail Interrupt
+    end
+
+    def load_celluloid
+      fail "Celluloid cannot be required until here, or it will break Toiler's daemonization" if defined?(::Celluloid) && Toiler.options[:daemon]
+
+      # Celluloid can't be loaded until after we've daemonized
+      # because it spins up threads and creates locks which get
+      # into a very bad state if forked.
+      require 'celluloid/autostart'
+      Celluloid.logger = (Toiler.options[:verbose] ? Toiler.logger : nil)
+      require 'toiler/supervisor'
+    end
+
+    def daemonize
+      return unless Toiler.options[:daemon]
+
+      fail ArgumentError, "You really should set a logfile if you're going to daemonize" unless Toiler.options[:logfile]
+
+      files_to_reopen = []
+      ObjectSpace.each_object(File) do |file|
+        files_to_reopen << file unless file.closed?
+      end
+
+      Process.daemon(true, true)
+
+      files_to_reopen.each do |file|
+        begin
+          file.reopen file.path, 'a+'
+          #file.sync = true
+        rescue ::Exception
+        end
+      end
+
+      [$stdout, $stderr].each do |io|
+        File.open(Toiler.options[:logfile], 'ab') do |f|
+          io.reopen(f)
+        end
+        #io.sync = true
+      end
+      $stdin.reopen('/dev/null')
+    end
+
+    def write_pid
+      if (path = Toiler.options[:pidfile])
+        File.open(path, 'w') do |f|
+          f.puts Process.pid
+        end
+      end
+    end
+
+    def parse_cli_args(argv)
+      opts = { queues: [] }
+
+      @parser = OptionParser.new do |o|
+        o.on '-d', '--daemon', 'Daemonize process' do |arg|
+          opts[:daemon] = arg
+        end
+
+        o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
+          opts[:require] = arg
+        end
+
+        o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
+          opts[:config_file] = arg
+        end
+
+        o.on '-R', '--rails', 'Load Rails' do |arg|
+          opts[:rails] = arg
+        end
+
+        o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
+          opts[:logfile] = arg
+        end
+
+        o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
+          opts[:pidfile] = arg
+        end
+
+        o.on '-v', '--verbose', 'Print more verbose output' do |arg|
+          opts[:verbose] = arg
+        end
+      end
+
+      @parser.banner = 'toiler [options]'
+      @parser.on_tail '-h', '--help', 'Show help' do
+        Toiler.logger.info @parser
+        exit 1
+      end
+      @parser.parse!(argv)
+      opts
+    end
+  end
+end

data/lib/toiler/core_ext.rb

@@ -0,0 +1,47 @@
+begin
+  require 'active_support/core_ext/hash/keys'
+  require 'active_support/core_ext/hash/deep_merge'
+rescue LoadError
+  class Hash
+    def stringify_keys
+      each_key do |key|
+        self[key.to_s] = delete(key)
+      end
+      self
+    end unless {}.respond_to?(:stringify_keys)
+
+    def symbolize_keys
+      each_key do |key|
+        self[(key.to_sym rescue key) || key] = delete(key)
+      end
+      self
+    end unless {}.respond_to?(:symbolize_keys)
+
+    def deep_symbolize_keys
+      each_key do |key|
+        value = delete(key)
+        self[(key.to_sym rescue key) || key] = value
+
+        value.deep_symbolize_keys if value.is_a? Hash
+      end
+      self
+    end unless {}.respond_to?(:deep_symbolize_keys)
+  end
+end
+
+begin
+  require 'active_support/core_ext/string/inflections'
+rescue LoadError
+  class String
+    def constantize
+      names = split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      constant = Object
+      names.each do |name|
+        constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+      end
+      constant
+    end
+  end unless ''.respond_to?(:constantize)
+end

data/lib/toiler/environment_loader.rb

@@ -0,0 +1,82 @@
+require 'erb'
+require 'yaml'
+
+module Toiler
+  class EnvironmentLoader
+    attr_reader :options
+
+    def self.load(options)
+      new(options).load
+    end
+
+    def self.load_for_rails_console
+      load(config_file: (Rails.root + 'config' + 'toiler.yml'))
+    end
+
+    def initialize(options)
+      @options = options
+    end
+
+    def load
+      initialize_logger
+      load_rails if options[:rails]
+      require_workers if options[:require]
+      Toiler.options.merge!(config_file_options)
+      Toiler.options.merge!(options)
+      initialize_aws
+    end
+
+    private
+
+    def config_file_options
+      if (path = options[:config_file])
+        unless File.exist?(path)
+          Toiler.logger.warn "Config file #{path} does not exist"
+          path = nil
+        end
+      end
+
+      return {} unless path
+
+      YAML.load(ERB.new(IO.read(path)).result).deep_symbolize_keys
+    end
+
+    def initialize_aws
+      # aws-sdk tries to load the credentials from the ENV variables: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
+      # when not explicit supplied
+      fail 'AWS Credentials needed!' if Toiler.options[:aws].empty? && (ENV['AWS_ACCESS_KEY_ID'].nil? || ENV['AWS_SECRET_ACCESS_KEY'].nil?)
+      return if Toiler.options[:aws].empty?
+
+      ::Aws.config[:region] = Toiler.options[:aws][:region]
+      ::Aws.config[:credentials] = ::Aws::Credentials.new Toiler.options[:aws][:access_key_id], Toiler.options[:aws][:secret_access_key]
+    end
+
+    def initialize_logger
+      Toiler::Logging.initialize_logger(options[:logfile]) if options[:logfile]
+      Toiler.logger.level = Logger::DEBUG if options[:verbose]
+    end
+
+    def load_rails
+      # Adapted from: https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/cli.rb
+
+      require 'rails'
+      if ::Rails::VERSION::MAJOR < 4
+        require File.expand_path('config/environment.rb')
+        ::Rails.application.eager_load!
+      else
+        # Painful contortions, see 1791 for discussion
+        require File.expand_path('config/application.rb')
+        ::Rails::Application.initializer 'toiler.eager_load' do
+          ::Rails.application.config.eager_load = true
+        end
+        require File.expand_path('config/environment.rb')
+      end
+
+      Toiler.logger.info 'Rails environment loaded'
+    end
+
+    def require_workers
+      require options[:require]
+    end
+  end
+end

data/lib/toiler/fetcher.rb

@@ -0,0 +1,39 @@
+module Toiler
+  class Fetcher
+    include Celluloid
+    include Celluloid::Logger
+
+    FETCH_LIMIT = 10.freeze
+
+    attr_accessor :queue, :wait, :batch
+
+    finalizer :shutdown
+
+    def initialize(queue, client = nil)
+      @queue = Queue.new queue, client
+      @wait = Toiler.options[:wait] || 20
+      @batch = Toiler.worker_class_registry[queue].batch?
+      async.poll_messages
+    end
+
+    def shutdown
+      instance_variables.each { |iv| remove_instance_variable iv }
+    end
+
+    def poll_messages
+      # AWS limits the batch size by 10
+      options = {
+        message_attribute_names: %w(All),
+        wait_time_seconds: wait
+      }
+
+      loop do
+        count = Toiler.manager.free_processors queue.name
+        options[:max_number_of_messages] = (batch || count > FETCH_LIMIT) ? FETCH_LIMIT : count
+        msgs = queue.receive_messages options
+        Toiler.manager.assign_messages queue.name, msgs unless msgs.empty?
+        Toiler.manager.wait_for_available_processors queue.name
+      end
+    end
+  end
+end

data/lib/toiler/logging.rb

@@ -0,0 +1,42 @@
+require 'time'
+require 'logger'
+
+module Toiler
+  module Logging
+    class Pretty < Logger::Formatter
+      # Provide a call() method that returns the formatted message.
+      def call(severity, time, _program_name, message)
+        "#{time.utc.iso8601} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
+      end
+
+      def context
+        c = Thread.current[:toiler_context]
+        c ? " #{c}" : ''
+      end
+    end
+
+    module_function
+
+    def with_context(msg)
+      Thread.current[:toiler_context] = msg
+      yield
+    ensure
+      Thread.current[:toiler_context] = nil
+    end
+
+    def initialize_logger(log_target = STDOUT)
+      @logger = Logger.new(log_target)
+      @logger.level = Logger::INFO
+      @logger.formatter = Pretty.new
+      @logger
+    end
+
+    def logger
+      @logger || initialize_logger
+    end
+
+    def logger=(log)
+      @logger = (log ? log : Logger.new('/dev/null'))
+    end
+  end
+end

data/lib/toiler/manager.rb

@@ -0,0 +1,109 @@
+require 'toiler/fetcher'
+require 'toiler/processor'
+
+module Toiler
+  class Manager
+    include Celluloid
+    include Celluloid::Logger
+
+    attr_accessor :queues, :client
+
+    finalizer :shutdown
+
+    def initialize
+      Toiler.set_manager current_actor
+      async.init
+    end
+
+    def init
+      @queues = Toiler.worker_class_registry
+      @client = ::Aws::SQS::Client.new
+      init_workers
+      init_conditions
+      pool_processors
+      supervise_fetchers
+    end
+
+    def shutdown
+      instance_variables.each { |iv| remove_instance_variable iv }
+    end
+
+    def stop
+      terminate_fetchers
+      terminate_processors
+    end
+
+    def processor_finished(queue)
+      @conditions[queue].broadcast
+    end
+
+    def init_workers
+      Toiler.worker_class_registry.each do |q, klass|
+        Toiler.worker_registry[q] = klass.new
+      end
+    end
+
+    def supervise_fetchers
+      queues.each do |queue, _klass|
+        Toiler.set_fetcher queue, Fetcher.supervise(queue, client).actors.first
+      end
+    end
+
+    def pool_processors
+      queues.each do |q, klass|
+        count = klass.concurrency
+        processor = if count > 1
+                      Processor.pool args: [q], size: count
+                    else
+                      Processor.supervise(q).actors.first
+                    end
+        Toiler.set_processor_pool q, processor
+      end
+    end
+
+    def terminate_fetchers
+      queues.each do |queue, _klass|
+        fetcher = Toiler.fetcher(queue)
+        fetcher.terminate if fetcher && fetcher.alive?
+      end
+    end
+
+    def terminate_processors
+      queues.each do |queue, _klass|
+        processor_pool = Toiler.processor_pool(queue)
+        processor_pool.terminate if processor_pool && processor_pool.alive?
+      end
+    end
+
+    def init_conditions
+      @conditions = {}
+      queues.each do |queue, _klass|
+        @conditions[queue] = Celluloid::Condition.new
+      end
+    end
+
+    def free_processors(queue)
+      return 1 unless Toiler.processor_pool(queue).respond_to? :idle_size
+      Toiler.processor_pool(queue).idle_size
+    end
+
+    def assign_messages(queue, messages)
+      processor_pool = Toiler.processor_pool(queue)
+      if batch? queue
+        processor_pool.async.process(queue, messages)
+      else
+        messages.each do |m|
+          processor_pool.async.process(queue, m)
+        end
+      end
+    end
+
+    def wait_for_available_processors(queue)
+      @conditions[queue].wait if free_processors(queue) == 0
+    end
+
+    def batch?(queue)
+      Toiler.worker_class_registry[queue].batch?
+    end
+  end
+end

data/lib/toiler/message.rb

@@ -0,0 +1,60 @@
+module Toiler
+  class Message
+    attr_accessor :client, :queue_url, :data
+
+    def initialize(client, queue_url, data)
+      @client = client
+      @queue_url = queue_url
+      @data = data
+    end
+
+    def delete
+      client.delete_message(
+        queue_url: queue_url,
+        receipt_handle: data.receipt_handle
+      )
+    end
+
+    def change_visibility(options)
+      client.change_message_visibility(
+        options.merge(queue_url: queue_url, receipt_handle: data.receipt_handle)
+      )
+    end
+
+    def visibility_timeout=(timeout)
+      client.change_message_visibility(
+        queue_url: queue_url,
+        receipt_handle: data.receipt_handle,
+        visibility_timeout: timeout
+      )
+    end
+
+    def message_id
+      data.message_id
+    end
+
+    def receipt_handle
+      data.receipt_handle
+    end
+
+    def md5_of_body
+      data.md5_of_body
+    end
+
+    def body
+      data.body
+    end
+
+    def attributes
+      data.attributes
+    end
+
+    def md5_of_message_attributes
+      data.md5_of_message_attributes
+    end
+
+    def message_attributes
+      data.message_attributes
+    end
+  end
+end

data/lib/toiler/processor.rb

@@ -0,0 +1,88 @@
+require 'json'
+require 'toiler/scheduler'
+
+module Toiler
+  class Processor
+    include Celluloid
+    include Celluloid::Logger
+
+    attr_accessor :queue, :scheduler
+
+    finalizer :shutdown
+
+    def initialize(queue)
+      @queue = queue
+      async.init
+    end
+
+    def init
+      @scheduler = Scheduler.supervise.actors.first
+      processor_finished
+    end
+
+    def shutdown
+      scheduler.terminate if scheduler && scheduler.alive?
+      instance_variables.each { |iv| remove_instance_variable iv }
+    end
+
+    def process(queue, sqs_msg)
+      exclusive do
+        worker = Toiler.worker_registry[queue]
+        timer = auto_visibility_timeout(queue, sqs_msg, worker.class)
+
+        begin
+          body = get_body(worker.class, sqs_msg)
+          worker.perform(sqs_msg, body)
+          sqs_msg.delete if worker.class.auto_delete?
+        ensure
+          timer.cancel if timer
+          ::ActiveRecord::Base.clear_active_connections!
+        end
+      end
+    ensure
+      processor_finished
+    end
+
+    def processor_finished
+      Toiler.manager.processor_finished queue
+    end
+
+    private
+
+    def auto_visibility_timeout(queue, sqs_msg, worker_class)
+      return unless worker_class.auto_visibility_timeout?
+      queue_visibility_timeout = Toiler.fetcher(queue).queue.visibility_timeout
+      block = lambda do |msg, visibility_timeout|
+        msg.visibility_timeout = visibility_timeout
+      end
+
+      scheduler.custom_every(queue_visibility_timeout - 5, sqs_msg, queue_visibility_timeout, block)
+    end
+
+    def get_body(worker_class, sqs_msg)
+      if sqs_msg.is_a? Array
+        sqs_msg.map { |m| parse_body(worker_class, m) }
+      else
+        parse_body(worker_class, sqs_msg)
+      end
+    end
+
+    def parse_body(worker_class, sqs_msg)
+      body_parser = worker_class.get_toiler_options[:parser]
+
+      case body_parser
+      when :json
+        JSON.parse(sqs_msg.body)
+      when Proc
+        body_parser.call(sqs_msg)
+      when :text, nil
+        sqs_msg.body
+      else
+        body_parser.load(sqs_msg.body) if body_parser.respond_to?(:load) # i.e. Oj.load(...) or MultiJson.load(...)
+      end
+    rescue => e
+      logger.error "Error parsing the message body: #{e.message}\nbody_parser: #{body_parser}\nsqs_msg.body: #{sqs_msg.body}"
+      nil
+    end
+  end
+end

data/lib/toiler/queue.rb

@@ -0,0 +1,53 @@
+module Toiler
+  class Queue
+    attr_accessor :name, :client, :url
+
+    def initialize(name, client = nil)
+      @name   = name
+      @client = client || ::Aws::SQS::Client.new
+      @url    = client.get_queue_url(queue_name: name).queue_url
+    end
+
+    def visibility_timeout
+      client.get_queue_attributes(
+        queue_url: url,
+        attribute_names: ['VisibilityTimeout']
+      ).attributes['VisibilityTimeout'].to_i
+    end
+
+    def delete_messages(options)
+      client.delete_message_batch(options.merge(queue_url: url))
+    end
+
+    def send_message(options)
+      client.send_message(sanitize_message_body(options.merge(queue_url: url)))
+    end
+
+    def send_messages(options)
+      client.send_message_batch(sanitize_message_body(options.merge(queue_url: url)))
+    end
+
+    def receive_messages(options)
+      client.receive_message(options.merge(queue_url: url))
+        .messages
+        .map { |m| Message.new(client, url, m) }
+    end
+
+    private
+
+    def sanitize_message_body(options)
+      messages = options[:entries] || [options]
+
+      messages.each do |m|
+        body = m[:message_body]
+        if body.is_a?(Hash)
+          m[:message_body] = JSON.dump(body)
+        elsif !body.is_a? String
+          fail ArgumentError, "The message body must be a String and you passed a #{body.class}"
+        end
+      end
+
+      options
+    end
+  end
+end

data/lib/toiler/scheduler.rb

@@ -0,0 +1,16 @@
+module Toiler
+  class Scheduler
+    include Celluloid
+    include Celluloid::Logger
+
+    execute_block_on_receiver :custom_every
+
+    def custom_every(*args, block)
+      period = args[0]
+      block_args = args[1..-1]
+      every(period) do
+        block.call(*block_args)
+      end
+    end
+  end
+end

data/lib/toiler/supervisor.rb

@@ -0,0 +1,23 @@
+require 'toiler/manager'
+
+module Toiler
+  class Supervisor < Celluloid::SupervisionGroup
+    include Celluloid
+
+    finalizer :shutdown
+
+    def initialize
+      @manager = Manager.new
+    end
+
+    def stop
+      @manager.stop
+      @manager.terminate if @manager.alive?
+    end
+
+    def shutdown
+      @manager.terminate if @manager.alive?
+      instance_variables.each { |iv| remove_instance_variable iv }
+    end
+  end
+end

data/lib/toiler/version.rb

@@ -0,0 +1,3 @@
+module Toiler
+  VERSION = '0.1.0'
+end

data/lib/toiler/worker.rb

@@ -0,0 +1,42 @@
+module Toiler
+  module Worker
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      def toiler_options(options)
+        if @toiler_options
+          @toiler_options = @toiler_options.merge options
+        else
+          @toiler_options = Toiler.default_options.merge options
+        end
+        Toiler.worker_class_registry[options[:queue]] = self if options[:queue]
+      end
+
+      def get_toiler_options
+        @toiler_options
+      end
+
+      def batch?
+        @toiler_options[:batch]
+      end
+
+      def concurrency
+        @toiler_options[:concurrency]
+      end
+
+      def queue
+        @toiler_options[:queue]
+      end
+
+      def auto_visibility_timeout?
+        @toiler_options[:auto_visibility_timeout]
+      end
+
+      def auto_delete?
+        @toiler_options[:auto_delete]
+      end
+    end
+  end
+end

data/toiler.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'toiler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'toiler'
+  spec.version       = Toiler::VERSION
+  spec.authors       = ['Sebastian Schepens']
+  spec.email         = ['sebas.schep@hotmail.com']
+  spec.description = spec.summary = 'Toiler is a super efficient AWS SQS thread based message processor'
+  spec.homepage      = 'https://github.com/sschepens/toiler'
+  spec.license       = 'LGPLv3'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables << 'toiler'
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'nokogiri'
+  spec.add_development_dependency 'dotenv'
+
+  spec.add_dependency 'aws-sdk', '~> 2.0.21'
+  spec.add_dependency 'celluloid', '~> 0.16.0'
+end

metadata

@@ -0,0 +1,179 @@
+--- !ruby/object:Gem::Specification
+name: toiler
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Sebastian Schepens
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.21
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.0.21
+- !ruby/object:Gem::Dependency
+  name: celluloid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.0
+description: Toiler is a super efficient AWS SQS thread based message processor
+email:
+- sebas.schep@hotmail.com
+executables:
+- toiler
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/toiler
+- lib/toiler.rb
+- lib/toiler/cli.rb
+- lib/toiler/core_ext.rb
+- lib/toiler/environment_loader.rb
+- lib/toiler/fetcher.rb
+- lib/toiler/logging.rb
+- lib/toiler/manager.rb
+- lib/toiler/message.rb
+- lib/toiler/processor.rb
+- lib/toiler/queue.rb
+- lib/toiler/scheduler.rb
+- lib/toiler/supervisor.rb
+- lib/toiler/version.rb
+- lib/toiler/worker.rb
+- toiler.gemspec
+homepage: https://github.com/sschepens/toiler
+licenses:
+- LGPLv3
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Toiler is a super efficient AWS SQS thread based message processor
+test_files: []