kicks 3.0.0.pre

data/examples/newrelic_metrics_worker.rb

@@ -0,0 +1,40 @@
+$: << File.expand_path('../lib', File.dirname(__FILE__))
+require 'sneakers'
+require 'sneakers/runner'
+require 'sneakers/metrics/newrelic_metrics'
+require 'open-uri'
+require 'newrelic_rpm'
+
+# With this configuration will send two types of data to newrelic server:
+# 1. Transaction data which you would see under  'Applications'
+# 2. Metrics where you will be able to see by configuring a dashboardi, available for enterprise accounts
+#
+# You should have newrelic.yml in the 'config' folder with the proper account settings
+
+Sneakers::Metrics::NewrelicMetrics.eagent ::NewRelic
+Sneakers.configure metrics: Sneakers::Metrics::NewrelicMetrics.new
+
+class MetricsWorker
+  include Sneakers::Worker
+  include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation
+
+  from_queue 'downloads'
+
+  def work(msg)
+    title = extract_title(open(msg))
+    logger.info "FOUND <#{title}>"
+    ack!
+  end
+
+  add_transaction_tracer :work, name: 'MetricsWorker', params: 'args[0]', category: :task
+
+  private
+
+  def extract_title(html)
+    html =~ /<title>(.*?)<\/title>/
+    $1
+  end
+end
+
+r = Sneakers::Runner.new([ MetricsWorker ])
+r.run

data/examples/profiling_worker.rb

@@ -0,0 +1,69 @@
+$: << File.expand_path('../lib', File.dirname(__FILE__))
+require 'sneakers'
+require 'sneakers/runner'
+require 'logger'
+
+
+profiling = ARGV[0]
+messages = 100_000
+
+
+if profiling
+  require 'ruby-prof'
+  messages /= 100 # profiling makes everything much slower (around 300req/s)
+end
+
+Sneakers.configure
+Sneakers.logger.level = Logger::ERROR
+
+Sneakers::Worker.configure_logger(Logger.new('/dev/null'))
+
+puts "feeding messages in"
+messages.times {
+  Sneakers.publish("{}", :to_queue => 'downloads')
+}
+puts "done"
+
+
+class ProfilingWorker
+  include Sneakers::Worker
+  from_queue 'downloads',
+             :ack => true,
+             :threads => 50,
+             :prefetch => 50,
+             :timeout_job_after => 1,
+             :exchange => 'sneakers',
+             :heartbeat => 5,
+             :amqp_heartbeat => 10
+  def work(msg)
+    ack!
+  end
+end
+
+
+
+r = Sneakers::Runner.new([ProfilingWorker])
+
+# ctrl-c and Ruby 2.0 breaks signal handling
+# Sidekiq has same issues
+# https://github.com/mperham/sidekiq/issues/728
+#
+# so we use a timeout and a thread that kills profiling
+if profiling
+  puts "profiling start"
+  RubyProf.start
+
+
+  Thread.new do
+    sleep 10
+    puts "stopping profiler"
+    result = RubyProf.stop
+
+    # Print a flat profile to text
+    printer = RubyProf::FlatPrinter.new(result)
+    printer.print(STDOUT)
+    exit(0)
+  end
+end
+
+r.run

data/examples/sneakers.conf.rb.example

@@ -0,0 +1,11 @@
+workers 2
+amqp "amqp://guest:guest@localhost:55672"
+
+before_fork do
+  Sneakers::logger.info " ** im before-fork'en ** "
+end
+
+
+after_fork do
+  Sneakers::logger.info " !! im after forke'n !! "
+end

data/examples/title_scraper.rb

@@ -0,0 +1,36 @@
+require "sneakers"
+require 'open-uri'
+require 'logger'
+
+def compose_or_localhost(key)
+  Resolv::DNS.new.getaddress(key)
+rescue 
+  "localhost"
+end
+
+rmq_addr = compose_or_localhost("rabbitmq")
+
+Sneakers.configure :log => STDOUT, :amqp => "amqp://guest:guest@#{rmq_addr}:5672"
+Sneakers.logger.level = Logger::INFO
+
+class TitleScraper
+  include Sneakers::Worker
+
+  from_queue 'downloads'
+
+  def work(msg)
+    title = extract_title(open(msg))
+    logger.info "FOUND <#{title}>"
+    ack!
+  end
+
+  private
+
+  def extract_title(html)
+    html =~ /<title>(.*?)<\/title>/
+    $1
+  end
+end
+
+
+

data/examples/workflow_worker.rb

@@ -0,0 +1,23 @@
+$: << File.expand_path('../lib', File.dirname(__FILE__))
+require 'sneakers'
+
+class WorkflowWorker
+  include Sneakers::Worker
+  from_queue 'downloads',
+             exchange_options: { durable: false },
+             queue_options: { durable: false },
+             :ack => true,
+             :threads => 50,
+             :prefetch => 50,
+             :timeout_job_after => 1,
+             :exchange => 'dummy',
+             :heartbeat => 5,
+             :amqp_heartbeat => 10
+
+  def work(msg)
+    logger.info("Seriously, i'm DONE.")
+    publish "cleaned up", :to_queue => "foobar"
+    logger.info("Published to 'foobar'")
+    ack!
+  end
+end

data/kicks.gemspec

@@ -0,0 +1,44 @@
+#!/usr/bin/env gem build
+# encoding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sneakers/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = 'kicks'
+  gem.version       = Sneakers::VERSION
+  gem.authors       = ['Dotan Nahum', 'Michael Klishin']
+  gem.email         = ['michael@clojurewerkz.org']
+  gem.description   = %q( Fast background processing framework for Ruby and RabbitMQ )
+  gem.summary       = %q( Fast background processing framework for Ruby and RabbitMQ )
+  gem.homepage      = 'https://github.com/ruby-amqp/kicks'
+  gem.license       = 'MIT'
+  gem.required_ruby_version = Gem::Requirement.new(">= 2.5")
+
+  gem.files         = `git ls-files`.split($/).reject { |f| f == 'Gemfile.lock' }
+  gem.executables   = gem.files.grep(/^bin/).
+                        reject { |f| f =~ /^bin\/ci/ }.
+                        map { |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(/^(test|spec|features)\//)
+  gem.require_paths = ['lib']
+
+  gem.add_dependency 'serverengine', '~> 2.1'
+  gem.add_dependency 'bunny', '~> 2.19'
+  gem.add_dependency 'concurrent-ruby', '~> 1.0'
+  gem.add_dependency 'thor'
+  gem.add_dependency 'rake', '>= 12.3', '< 14.0'
+
+  # for integration environment (see .travis.yml and integration_spec)
+  gem.add_development_dependency 'rabbitmq_http_api_client'
+  gem.add_development_dependency 'redis'
+
+  gem.add_development_dependency 'minitest', '~> 5.15'
+  gem.add_development_dependency 'rr', '~> 3.0'
+  gem.add_development_dependency 'unparser', '~> 0.2'
+  gem.add_development_dependency 'simplecov', '~> 0.21'
+  gem.add_development_dependency 'simplecov-rcov-text'
+  gem.add_development_dependency 'guard', '~> 2.18'
+  gem.add_development_dependency 'guard-minitest', '~> 2.4'
+  gem.add_development_dependency 'pry-byebug', '~> 3.9'
+end

data/lib/sneakers/cli.rb

@@ -0,0 +1,122 @@
+require 'thor'
+require 'sneakers/runner'
+
+
+#
+# $ sneakers work TitleWorker,FooWorker
+# $ sneakers stop
+# $ sneakers recycle
+# $ sneakers reload
+# $ sneakers init
+#
+#
+module Sneakers
+  class CLI < Thor
+
+    SNEAKERS=<<-EOF
+
+        __
+    ,--'  >  Sneakers
+    `=====
+
+    EOF
+
+    BANNER = SNEAKERS
+
+    method_option :debug
+    method_option :daemonize
+    method_option :log
+    method_option :pid_path
+    method_option :require
+
+    desc "work FirstWorker,SecondWorker ... ,NthWorker", "Run workers"
+    def work(workers = "")
+      opts = {
+        :daemonize => !!options[:daemonize]
+      }
+
+      opts[:log] = options[:log] || (opts[:daemonize] ? 'sneakers.log' : STDOUT)
+      opts[:pid_path] = options[:pid_path] if options[:pid_path]
+
+      if opts[:daemonize]
+        puts "*** DEPRACATED: self-daemonization '--daemonize' is considered a bad practice, which is why this feature will be removed in future versions. Please run Sneakers in front, and use things like upstart, systemd, or supervisor to manage it as a daemon."
+      end
+
+
+      Sneakers.configure(opts)
+
+      require_boot File.expand_path(options[:require]) if options[:require]
+
+      if workers.empty?
+        workers = Sneakers::Worker::Classes
+      else
+        workers, missing_workers = Sneakers::Utils.parse_workers(workers)
+      end
+
+      unless missing_workers.nil? || missing_workers.empty?
+        say "Missing workers: #{missing_workers.join(', ')}" if missing_workers
+        say "Did you `require` properly?"
+        return
+      end
+
+      if workers.empty?
+        say <<-EOF
+        Error: No workers found.
+        Please require your worker classes before specifying in CLI
+
+          $ sneakers work FooWorker
+                          ^- require this in your code
+
+        EOF
+        return
+      end
+
+      r = Sneakers::Runner.new(workers)
+
+      pid = Sneakers::CONFIG[:pid_path]
+
+      say SNEAKERS
+      say "Workers ....: #{em workers.join(', ')}"
+      say "Log ........: #{em (Sneakers::CONFIG[:log] == STDOUT ? 'Console' : Sneakers::CONFIG[:log]) }"
+      say "PID ........: #{em pid}"
+      say ""
+      say (" "*31)+"Process control"
+      say "="*80
+      say "Stop (nicely) ..............: kill -SIGTERM `cat #{pid}`"
+      say "Stop (immediate) ...........: kill -SIGQUIT `cat #{pid}`"
+      say "Restart (nicely) ...........: kill -SIGUSR1 `cat #{pid}`"
+      say "Restart (immediate) ........: kill -SIGHUP `cat #{pid}`"
+      say "Reconfigure ................: kill -SIGUSR2 `cat #{pid}`"
+      say "Scale workers ..............: reconfigure, then restart"
+      say "="*80
+      say ""
+
+      if options[:debug]
+        say "==== configuration ==="
+        say Sneakers::CONFIG.inspect
+        say "======================"
+      end
+
+      r.run
+    end
+
+
+  private
+    def require_boot(file)
+      load file
+    end
+
+    def em(text)
+      shell.set_color(text, nil, true)
+    end
+
+    def ok(detail=nil)
+      text = detail ? "OK, #{detail}." : "OK."
+      say text, :green
+    end
+
+    def error(detail)
+      say detail, :red
+    end
+  end
+end

data/lib/sneakers/concerns/logging.rb

@@ -0,0 +1,34 @@
+module Sneakers
+  module Concerns
+    module Logging
+      def self.included(base)
+        base.extend ClassMethods
+        base.send :define_method, :logger do
+          base.logger
+        end
+      end
+
+      module ClassMethods
+        def logger
+          @logger
+        end
+
+        def logger=(logger)
+          @logger = logger
+        end
+
+        def configure_logger(log=nil)
+          if log
+            @logger = log
+          else
+            @logger = Logger.new(STDOUT)
+            @logger.level = Logger::INFO
+            @logger.formatter = Sneakers::Support::ProductionFormatter
+          end
+        end
+      end
+    end
+  end
+end
+
+

data/lib/sneakers/concerns/metrics.rb

@@ -0,0 +1,34 @@
+require 'sneakers/metrics/null_metrics'
+
+module Sneakers
+  module Concerns
+    module Metrics
+      def self.included(base)
+        base.extend ClassMethods
+        base.send :define_method, :metrics do
+          base.metrics
+        end
+      end
+
+      module ClassMethods
+        def metrics
+          @metrics
+        end
+
+        def metrics=(metrics)
+          @metrics = metrics
+        end
+
+        def configure_metrics(metrics=nil)
+          if metrics
+            @metrics = metrics
+          else
+            @metrics = Sneakers::Metrics::NullMetrics.new
+          end
+        end
+      end
+    end
+  end
+end
+
+

data/lib/sneakers/configuration.rb

@@ -0,0 +1,125 @@
+require 'sneakers/error_reporter'
+require 'forwardable'
+
+module Sneakers
+  class Configuration
+
+    extend Forwardable
+    def_delegators :@hash, :to_hash, :[], :[]=, :==, :fetch, :delete, :has_key?
+
+    EXCHANGE_OPTION_DEFAULTS = {
+      :type               => :direct,
+      :durable            => true,
+      :auto_delete        => false,
+      :arguments => {} # Passed as :arguments to Bunny::Channel#exchange
+    }.freeze
+
+    QUEUE_OPTION_DEFAULTS = {
+      :durable            => true,
+      :auto_delete        => false,
+      :exclusive          => false,
+      :arguments => {}
+    }.freeze
+
+    DEFAULTS = {
+      # Set up default handler which just logs the error.
+      # Remove this in production if you don't want sensitive data logged.
+      :error_reporters => [Sneakers::ErrorReporter::DefaultLogger.new],
+
+      # runner
+      :runner_config_file => nil,
+      :metrics            => nil,
+      :daemonize          => false,
+      :start_worker_delay => 0.2,
+      :workers            => 4,
+      :log                => STDOUT,
+      :pid_path           => 'sneakers.pid',
+      :amqp_heartbeat     => 30,
+
+      # workers
+      :prefetch           => 10,
+      :threads            => 10,
+      :share_threads      => false,
+      :ack                => true,
+      :heartbeat          => 30,
+      :hooks              => {},
+      :exchange           => 'sneakers',
+      :exchange_options   => EXCHANGE_OPTION_DEFAULTS,
+      :queue_options      => QUEUE_OPTION_DEFAULTS
+    }.freeze
+
+
+    def initialize
+      clear
+    end
+
+    def clear
+      @hash = DEFAULTS.dup
+      @hash[:amqp]  = ENV.fetch('RABBITMQ_URL', 'amqp://guest:guest@localhost:5672')
+      @hash[:vhost] = AMQ::Settings.parse_amqp_url(@hash[:amqp]).fetch(:vhost, '/')
+    end
+
+    def merge!(hash)
+      hash = hash.dup
+      hash = map_all_deprecated_options(hash)
+
+      # parse vhost from amqp if vhost is not specified explicitly, only
+      # if we're not given a connection to use.
+      if hash[:connection].nil?
+        if hash[:vhost].nil? && !hash[:amqp].nil?
+          hash[:vhost] = AMQ::Settings.parse_amqp_url(hash[:amqp]).fetch(:vhost, '/')
+        end
+      else
+        # If we are given a Bunny object, ignore params we'd otherwise use to
+        # create one.  This removes any question about where config params are
+        # coming from, and makes it more likely that downstream code that needs
+        # this info gets it from the right place.
+        [:vhost, :amqp, :heartbeat].each do |k|
+          hash.delete(k)
+          @hash.delete(k)
+        end
+      end
+
+      @hash = deep_merge(@hash, hash)
+    end
+
+    def merge(hash)
+      instance = self.class.new
+      instance.merge! to_hash
+      instance.merge! hash
+      instance
+    end
+
+    def inspect_with_redaction
+      redacted = self.class.new
+      redacted.merge! to_hash
+
+      # redact passwords
+      redacted[:amqp] = redacted[:amqp].sub(/(?<=\Aamqp:\/)[^@]+(?=@)/, "<redacted>") if redacted.has_key?(:amqp)
+      return redacted.inspect_without_redaction
+    end
+    alias_method :inspect_without_redaction, :inspect
+    alias_method :inspect, :inspect_with_redaction
+
+    def map_all_deprecated_options(hash)
+      hash = map_deprecated_options_key(:exchange_options, :exchange_type, :type, true, hash)
+      hash = map_deprecated_options_key(:exchange_options, :exchange_arguments, :arguments, true, hash)
+      hash = map_deprecated_options_key(:exchange_options, :durable, :durable, false, hash)
+      hash = map_deprecated_options_key(:queue_options, :durable, :durable, true, hash)
+      hash = map_deprecated_options_key(:queue_options, :arguments, :arguments, true, hash)
+      hash
+    end
+
+    def map_deprecated_options_key(target_key, deprecated_key, key, delete_deprecated_key, hash = {})
+      return hash if hash[deprecated_key].nil?
+      hash = deep_merge({ target_key => { key => hash[deprecated_key] } }, hash)
+      hash.delete(deprecated_key) if delete_deprecated_key
+      hash
+    end
+
+    def deep_merge(first, second)
+      merger = proc { |_, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 }
+      first.merge(second, &merger)
+    end
+  end
+end

data/lib/sneakers/content_encoding.rb

@@ -0,0 +1,47 @@
+module Sneakers
+  class ContentEncoding
+    def self.register(content_encoding: nil, encoder: nil, decoder: nil)
+      # This can be removed when support is dropped for ruby 2.0 and replaced
+      # by a keyword arg with no default value
+      fail ArgumentError, 'missing keyword: content_encoding' if content_encoding.nil?
+      fail ArgumentError, 'missing keyword: encoder' if encoder.nil?
+      fail ArgumentError, 'missing keyword: decoder' if decoder.nil?
+
+      fail ArgumentError, "#{content_encoding} encoder must be a proc" unless encoder.is_a? Proc
+      fail ArgumentError, "#{content_encoding} decoder must be a proc" unless decoder.is_a? Proc
+
+      fail ArgumentError, "#{content_encoding} encoder must accept one argument, the payload" unless encoder.arity == 1
+      fail ArgumentError, "#{content_encoding} decoder must accept one argument, the payload" unless decoder.arity == 1
+      @_encodings[content_encoding] = new(encoder, decoder)
+    end
+
+    def self.encode(payload, content_encoding)
+      return payload unless content_encoding
+      @_encodings[content_encoding].encoder.(payload)
+    end
+
+    def self.decode(payload, content_encoding)
+      return payload unless content_encoding
+      @_encodings[content_encoding].decoder.(payload)
+    end
+
+    def self.reset!
+      @_encodings = Hash.new(
+        new(passthrough, passthrough)
+      )
+    end
+
+    def self.passthrough
+      ->(payload) { payload }
+    end
+
+    def initialize(encoder, decoder)
+      @encoder = encoder
+      @decoder = decoder
+    end
+
+    attr_reader :encoder, :decoder
+
+    reset!
+  end
+end

data/lib/sneakers/content_type.rb

@@ -0,0 +1,47 @@
+module Sneakers
+  class ContentType
+    def self.register(content_type: nil, serializer: nil, deserializer: nil)
+      # This can be removed when support is dropped for ruby 2.0 and replaced
+      # by a keyword arg with no default value
+      fail ArgumentError, 'missing keyword: content_type' if content_type.nil?
+      fail ArgumentError, 'missing keyword: serializer' if serializer.nil?
+      fail ArgumentError, 'missing keyword: deserializer' if deserializer.nil?
+
+      fail ArgumentError, "#{content_type} serializer must be a proc" unless serializer.is_a? Proc
+      fail ArgumentError, "#{content_type} deserializer must be a proc" unless deserializer.is_a? Proc
+
+      fail ArgumentError, "#{content_type} serializer must accept one argument, the payload" unless serializer.arity == 1
+      fail ArgumentError, "#{content_type} deserializer must accept one argument, the payload" unless deserializer.arity == 1
+      @_types[content_type] = new(serializer, deserializer)
+    end
+
+    def self.serialize(payload, content_type)
+      return payload unless content_type
+      @_types[content_type].serializer.(payload)
+    end
+
+    def self.deserialize(payload, content_type)
+      return payload unless content_type
+      @_types[content_type].deserializer.(payload)
+    end
+
+    def self.reset!
+      @_types = Hash.new(
+        new(passthrough, passthrough)
+      )
+    end
+
+    def self.passthrough
+      ->(payload) { payload }
+    end
+
+    def initialize(serializer, deserializer)
+      @serializer = serializer
+      @deserializer = deserializer
+    end
+
+    attr_reader :serializer, :deserializer
+
+    reset!
+  end
+end

data/lib/sneakers/error_reporter.rb

@@ -0,0 +1,33 @@
+# Ripped off from https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/exception_handler.rb
+module Sneakers
+  module ErrorReporter
+    class DefaultLogger
+      def call(exception, worker, context_hash)
+        Sneakers.logger.warn(context_hash) unless context_hash.empty?
+        log_string = ''
+        log_string += "[Exception error=#{exception.message.inspect} error_class=#{exception.class} worker_class=#{worker.class}"  unless exception.nil?
+        log_string += " backtrace=#{exception.backtrace.take(50).join(',')}" unless exception.nil? || exception.backtrace.nil?
+        log_string += ']'
+        Sneakers.logger.error log_string
+      end
+    end
+
+    def worker_error(exception, context_hash = {})
+      Sneakers.error_reporters.each do |handler|
+        begin
+          handler.call(exception, self, context_hash)
+        rescue SignalException, SystemExit
+          # ServerEngine handles these exceptions, so they are not expected
+          # to be raised within the error reporter.
+          # Nevertheless, they are listed here to ensure that they are not
+          # caught by the rescue block below.
+          raise
+        rescue Exception => inner_exception
+          Sneakers.logger.error '!!! ERROR REPORTER THREW AN ERROR !!!'
+          Sneakers.logger.error inner_exception
+          Sneakers.logger.error inner_exception.backtrace.join("\n") unless inner_exception.backtrace.nil?
+        end
+      end
+    end
+  end
+end

data/lib/sneakers/errors.rb

@@ -0,0 +1,2 @@
+module Sneakers
+end