shoryuken 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 720717277a58652fd893ef470426e5696b4943de
+  data.tar.gz: a5aad87ea2ffe866454cfa65ce78820c46def1a3
+SHA512:
+  metadata.gz: 302578e60de657fafee3faf3952adf70c848cd5aec1d124f6e34db78e82269ad25bd629f6ce25737a674804a2223f85eff812ad6262c0c6d9ffb6524a08c20f2
+  data.tar.gz: 17966390a08f67d9830e804bc0f24b11efb0658648e515dce18d0c52ccd72ec60d0457c431ecf4fd4d0905d34e29fae3ca5fc3244548b25f562eda14cc5039b8

data/.gitignore

@@ -0,0 +1,25 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+shoryuken.yml
+*.pid
+*.log

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in shoryuken.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,5 @@
+Copyright (c) Pablo Cantero
+
+Shoryuken is an Open Source project licensed under the terms of
+the LGPLv3 license.  Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
+for license text.

data/README.md

@@ -0,0 +1,135 @@
+# Shoryuken
+
+![](shoryuken.jpg)
+
+Shoryuken _sho-ryu-ken_ is a super efficient [AWS SQS](https://aws.amazon.com/sqs/) thread based message processor.
+
+## Key features
+
+### Load balancing
+
+Yeah, Shoryuken load balances the messages consumption, for example:
+
+Given this configuration:
+
+```yaml
+concurrency: 25,
+delay: 25,
+queues:
+  - [shoryuken, 6]
+  - [uppercut, 2]
+  - [sidekiq, 1]
+```
+
+And supposing all the queues are full of messages, the configuration above will make Shoryuken to process "shoryuken" 3 times more than "uppercut" and 6 times more than "sidekiq",
+splitting the work among the 25 available processors.
+
+If the "shoryuken" queue gets empty, Shoryuken will keep using the 25 processors, but only to process "uppercut" (2 times more than "sidekiq") and "sidekiq".
+
+If the "shoryuken" queue gets a new message, Shoryuken will smoothly increase back the "shoryuken" weight one by one until it reaches the weight of 5 again.
+
+If all queues get empty, all processors will be changed to the waiting state and the queues will be checked every `delay: 25`. If any queue gets a new message, Shoryuken will bring back the processors one by one to the ready state.
+
+### Fetch in batches
+
+To be even more performance and cost efficient, Shoryuken fetches SQS messages in batches.
+
+## Resque compatible?
+
+Shoryuken isn't Resque compatible, it passes the [original SQS message](http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/SQS/ReceivedMessage.html) to the workers.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'shoryuken'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install shoryuken
+
+## Usage
+
+### Worker class
+
+```ruby
+class HelloWorker
+  include Shoryuken::Worker
+
+  shoryuken_options queue: 'hello', auto_delete: true
+  # shoryuken_options queue: ->{ "#{ENV['environment']_hello" }, auto_delete: true
+
+  def perform(sqs_msg)
+    puts "Hello #{sqs_msg.body}"
+  end
+end
+```
+
+### Sending a message
+
+```ruby
+Shoryuken::Client.queues('hello').send_message('Pablo')
+```
+
+### Configuration
+
+Sample configuration file `shoryuken.yml`.
+
+```yaml
+aws:
+  access_key_id:      ...
+  secret_access_key:  ...
+  region:             us-east-1
+  receive_message:
+    attributes:
+      - receive_count
+      - sent_at
+concurrency: 25,
+delay: 25,
+timeout: 8
+queues:
+  - [shoryuken, 6]
+  - [uppercut, 2]
+  - [sidekiq, 1]
+```
+
+### Start Shoryuken
+
+```shell
+bundle exec shoryuken -r worker.rb -C shoryuken.yml
+```
+
+### Middleware
+
+```ruby
+class MyServerHook
+  def call(worker_instance, queue, sqs_msg)
+    puts 'Before work'
+    yield
+    puts 'After work'
+  end
+end
+
+Shoryuken.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add MyServerHook
+    # chain.remove MyServerHook
+  end
+end
+```
+
+## Credits
+
+[Mike Perham](https://github.com/mperham), creator of [Sidekiq](https://github.com/mperham/sidekiq), and [everybody who contributed to it](https://github.com/mperham/sidekiq/graphs/contributors). Shoryuken wouldn't exist as it is without those contributions.
+
+## Contributing
+
+1. Fork it ( https://github.com/phstc/shoryuken/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,41 @@
+require 'bundler/gem_tasks'
+$stdout.sync = true
+
+task :console do
+  require 'pry'
+  require 'shoryuken'
+
+  config_file = File.join File.expand_path('..', __FILE__), 'shoryuken.yml'
+
+  if File.exist? config_file
+    config = YAML.load File.read(config_file)
+
+    AWS.config(config['aws'])
+  end
+
+  def push(queue, message)
+    Shoryuken::Client.queues(queue).send_message message
+  end
+
+  ARGV.clear
+  Pry.start
+end
+
+task :push_test, :size do |t, args|
+  require 'yaml'
+  require 'shoryuken'
+
+  config = YAML.load File.read(File.join(File.expand_path('..', __FILE__), 'shoryuken.yml'))
+
+  AWS.config(config['aws'])
+
+  (args[:size] || 1).to_i.times.map do |i|
+    Thread.new do
+      Shoryuken::Client.queues('shoryuken').send_message("shoryuken #{i}")
+      Shoryuken::Client.queues('uppercut').send_message("uppercut #{i}")
+      Shoryuken::Client.queues('sidekiq').send_message("sidekiq #{i}")
+
+      puts "Push test ##{i + 1}"
+    end
+  end.each &:join
+end

data/bin/shoryuken

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require 'shoryuken'
+require 'shoryuken/cli'
+
+begin
+  Shoryuken::CLI.instance.run(ARGV)
+rescue => e
+  raise e if $DEBUG
+  STDERR.puts e.message
+  STDERR.puts e.backtrace.join("\n")
+  exit 1
+end

data/examples/all.rb

@@ -0,0 +1,5 @@
+$stdout.sync = true
+
+require_relative 'shoryuken_worker'
+require_relative 'uppercut_worker'
+require_relative 'sidekiq_worker'

data/examples/shoryuken_worker.rb

@@ -0,0 +1,11 @@
+class ShoryukenWorker
+  include Shoryuken::Worker
+
+  shoryuken_options queue: 'shoryuken', auto_delete: true
+
+  def perform(sqs_msg)
+    puts "Shoryuken: '#{sqs_msg.body}'"
+
+    sleep rand(0..1)
+  end
+end

data/examples/sidekiq_worker.rb

@@ -0,0 +1,11 @@
+class SidekiqWorker
+  include Shoryuken::Worker
+
+  shoryuken_options queue: 'sidekiq', auto_delete: true
+
+  def perform(sqs_msg)
+    puts "Sidekiq: '#{sqs_msg.body}'"
+
+    sleep rand(0..1)
+  end
+end

data/examples/uppercut_worker.rb

@@ -0,0 +1,11 @@
+class UppercutWorker
+  include Shoryuken::Worker
+
+  shoryuken_options queue: 'uppercut', auto_delete: true
+
+  def perform(sqs_msg)
+    puts "Uppercut: '#{sqs_msg.body}'"
+
+    sleep rand(0..1)
+  end
+end

data/lib/shoryuken.rb

@@ -0,0 +1,69 @@
+require 'yaml'
+require 'aws-sdk'
+require 'celluloid'
+require 'time'
+
+require 'shoryuken/version'
+require 'shoryuken/core_ext'
+require 'shoryuken/util'
+require 'shoryuken/manager'
+require 'shoryuken/processor'
+require 'shoryuken/fetcher'
+require 'shoryuken/client'
+require 'shoryuken/worker'
+require 'shoryuken/launcher'
+require 'shoryuken/logging'
+require 'shoryuken/middleware/chain'
+require 'shoryuken/middleware/server/auto_delete'
+require 'shoryuken/middleware/server/logging'
+
+module Shoryuken
+  DEFAULTS = {
+    concurrency: 25,
+    queues: [],
+    receive_message_options: {},
+    delay: 25,
+    timeout: 8
+  }
+
+  # { 'my_queue1' => Worker1
+  #   'my_queue2' => Worker2 }
+  @@workers = {}
+
+  @@queues = []
+
+  def self.options
+    @options ||= DEFAULTS.dup
+  end
+
+  def self.register_worker(queue, clazz)
+    @@workers[queue] = clazz
+  end
+
+  def self.workers
+    @@workers
+  end
+
+  def self.queues
+    @@queues
+  end
+
+  def self.logger
+    Shoryuken::Logging.logger
+  end
+
+  # Shoryuken.configure_server do |config|
+  #   config.server_middleware do |chain|
+  #     chain.add MyServerHook
+  #   end
+  # end
+  def self.configure_server
+    yield self
+  end
+
+  def self.server_middleware
+    @server_chain ||= Processor.default_middleware
+    yield @server_chain if block_given?
+    @server_chain
+  end
+end

data/lib/shoryuken/cli.rb

@@ -0,0 +1,222 @@
+$stdout.sync = true
+
+require 'singleton'
+require 'optparse'
+require 'erb'
+
+module Shoryuken
+  class CLI
+    include Util
+    include Singleton
+
+    attr_accessor :launcher
+
+    def run(args)
+      self_read, self_write = IO.pipe
+
+      %w(INT TERM USR1 USR2 TTIN).each do |sig|
+        trap sig do
+          self_write.puts(sig)
+        end
+      end
+
+      setup_options(args)
+      initialize_logger
+      validate!
+      daemonize
+      initialize_aws
+      require_workers
+      write_pid
+
+      @launcher = Shoryuken::Launcher.new
+
+      begin
+        launcher.run
+
+        while readable_io = IO.select([self_read])
+          signal = readable_io.first[0].gets.strip
+          handle_signal(signal)
+        end
+      rescue Interrupt
+        launcher.stop(shutdown: true)
+        exit 0
+      end
+    end
+
+    private
+
+    def daemonize
+      return unless Shoryuken.options[:daemon]
+
+      raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless Shoryuken.options[:logfile]
+
+      files_to_reopen = []
+      ObjectSpace.each_object(File) do |file|
+        files_to_reopen << file unless file.closed?
+      end
+
+      Process.daemon(true, true)
+
+      files_to_reopen.each do |file|
+        begin
+          file.reopen file.path, "a+"
+          file.sync = true
+        rescue ::Exception
+        end
+      end
+
+      [$stdout, $stderr].each do |io|
+        File.open(Shoryuken.options[:logfile], 'ab') do |f|
+          io.reopen(f)
+        end
+        io.sync = true
+      end
+      $stdin.reopen('/dev/null')
+
+      initialize_logger
+    end
+
+    def write_pid
+      if path = Shoryuken.options[:pidfile]
+        File.open(path, 'w') do |f|
+          f.puts Process.pid
+        end
+      end
+    end
+
+    def parse_options(argv)
+      opts = {}
+
+      @parser = OptionParser.new do |o|
+        o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
+          opts[:concurrency] = Integer(arg)
+        end
+
+        o.on '-d', '--daemon', 'Daemonize process' do |arg|
+          opts[:daemon] = arg
+        end
+
+        o.on '-q', '--queue QUEUE[,WEIGHT]...', 'Queues to process with optional weights' do |arg|
+          queues_and_weights = arg.scan(/([\w\.-]+),?(\d*)/)
+          parse_queues opts, queues_and_weights
+        end
+
+        o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
+          opts[:require] = arg
+        end
+
+        o.on '-C', '--config PATH', 'path to YAML config file' do |arg|
+          opts[:config_file] = arg
+        end
+
+        o.on '-L', '--logfile PATH', 'path to writable logfile' do |arg|
+          opts[:logfile] = arg
+        end
+
+        o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
+          opts[:pidfile] = arg
+        end
+
+        o.on '-v', '--verbose', 'Print more verbose output' do |arg|
+          opts[:verbose] = arg
+        end
+
+        o.on '-V', '--version', 'Print version and exit' do |arg|
+          puts "Shoryuken #{Shoryuken::VERSION}"
+          exit 0
+        end
+      end
+
+      @parser.banner = 'shoryuken [options]'
+      @parser.on_tail '-h', '--help', 'Show help' do
+        Shoryuken.logger.info @parser
+        exit 1
+      end
+      @parser.parse!(argv)
+      opts
+    end
+
+    def handle_signal(sig)
+      Shoryuken.logger.info "Got #{sig} signal"
+
+      case sig
+      when 'USR1'
+        Shoryuken.logger.info "Received USR1, will soft shutdown down"
+
+        launcher.stop
+
+        exit 0
+      when 'TTIN'
+        Thread.list.each do |thread|
+          Shoryuken.logger.info "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
+          if thread.backtrace
+            Shoryuken.logger.info thread.backtrace.join("\n")
+          else
+            Shoryuken.logger.info "<no backtrace available>"
+          end
+        end
+
+        Shoryuken.logger.info "Ready processors: #{launcher.manager.instance_variable_get(:@ready).size}"
+        Shoryuken.logger.info "Busy processors: #{launcher.manager.instance_variable_get(:@busy).size}"
+
+        launcher.manager.instance_variable_get(:@queues).inject({}) do |weights, queue|
+          weights[queue] = weights[queue].to_i + 1
+          weights
+        end.each do |queue, weight|
+          Shoryuken.logger.info "Current queue '#{queue}' weight: #{weight}"
+        end
+      else
+        Shoryuken.logger.info "Received #{sig}, will shutdown down"
+
+        raise Interrupt
+      end
+    end
+
+    def setup_options(args)
+      options = parse_options(args)
+
+      config = options[:config_file] ? parse_config(options[:config_file]).deep_symbolize_keys : {}
+
+      Shoryuken.options.merge!(config)
+
+      Shoryuken.options.merge!(options)
+
+      parse_queues
+    end
+
+    def parse_config(cfile)
+      opts = {}
+      if File.exist?(cfile)
+        opts = YAML.load(ERB.new(IO.read(cfile)).result)
+      end
+
+      opts
+    end
+
+    def initialize_logger
+      Shoryuken::Logging.initialize_logger(Shoryuken.options[:logfile]) if Shoryuken.options[:logfile]
+
+      Shoryuken.logger.level = Logger::DEBUG if Shoryuken.options[:verbose]
+    end
+
+    def validate!
+      raise ArgumentError, 'No queues supplied' if Shoryuken.queues.empty?
+    end
+
+    def initialize_aws
+      AWS.config Shoryuken.options[:aws] if Shoryuken.options[:aws]
+    end
+
+    def require_workers
+      require Shoryuken.options[:require] if Shoryuken.options[:require]
+    end
+
+    def parse_queues
+      Shoryuken.options[:queues].each { |queue_and_weight| parse_queue *queue_and_weight }
+    end
+
+    def parse_queue(queue, weight = nil)
+      [weight.to_i, 1].max.times { Shoryuken.queues << queue }
+    end
+  end
+end