sneakers_custom_bunny 1.0.4

data/examples/profiling_worker.rb

@@ -0,0 +1,69 @@
+$: << File.expand_path('../lib', File.dirname(__FILE__))
+require 'sneakers'
+require 'sneakers/runner'
+require 'logger'
+
+
+profiling = ARGV[0]
+messages = 100_000
+
+
+if profiling
+  require 'ruby-prof'
+  messages /= 100 # profiling makes everything much slower (around 300req/s)
+end
+
+Sneakers.configure
+Sneakers.logger.level = Logger::ERROR
+
+Sneakers::Worker.configure_logger(Logger.new('/dev/null'))
+
+puts "feeding messages in"
+messages.times {
+  Sneakers.publish("{}", :to_queue => 'downloads')
+}
+puts "done"
+
+
+class ProfilingWorker
+  include Sneakers::Worker
+  from_queue 'downloads',
+             :ack => true,
+             :threads => 50,
+             :prefetch => 50,
+             :timeout_job_after => 1,
+             :exchange => 'sneakers',
+             :heartbeat => 5
+  def work(msg)
+    ack!
+  end
+end
+
+
+
+r = Sneakers::Runner.new([ProfilingWorker])
+
+# ctrl-c and Ruby 2.0 breaks signal handling
+# Sidekiq has same issues
+# https://github.com/mperham/sidekiq/issues/728
+#
+# so we use a timeout and a thread that kills profiling
+if profiling
+  puts "profiling start"
+  RubyProf.start
+
+
+  Thread.new do
+    sleep 10
+    puts "stopping profiler"
+    result = RubyProf.stop
+
+    # Print a flat profile to text
+    printer = RubyProf::FlatPrinter.new(result)
+    printer.print(STDOUT)
+    exit(0)
+  end
+end
+
+r.run
+

data/examples/sneakers.conf.rb.example

@@ -0,0 +1,11 @@
+workers 2
+amqp "amqp://guest:guest@localhost:55672"
+
+before_fork do
+  Sneakers::logger.info " ** im before-fork'en ** "
+end
+
+
+after_fork do
+  Sneakers::logger.info " !! im after forke'n !! "
+end

data/examples/title_scraper.rb

@@ -0,0 +1,23 @@
+require "sneakers"
+require 'open-uri'
+require 'nokogiri'
+
+require 'logger'
+
+Sneakers.configure :log => STDOUT
+Sneakers.logger.level = Logger::INFO
+
+class TitleScraper
+  include Sneakers::Worker
+
+  from_queue 'downloads'
+
+  def work(msg)
+    doc = Nokogiri::HTML(open(msg))
+    worker_trace "FOUND <#{doc.css('title').text}>"
+    ack!
+  end
+end
+
+
+

data/examples/workflow_worker.rb

@@ -0,0 +1,23 @@
+$: << File.expand_path('../lib', File.dirname(__FILE__))
+require 'sneakers'
+
+class WorkflowWorker
+  include Sneakers::Worker
+  from_queue 'downloads',
+             :durable => false,
+             :ack => true,
+             :threads => 50,
+             :prefetch => 50,
+             :timeout_job_after => 1,
+             :exchange => 'dummy',
+             :heartbeat => 5
+
+  def work(msg)
+    logger.info("Seriously, i'm DONE.")
+    publish "cleaned up", :to_queue => "foobar"
+    logger.info("Published to 'foobar'")
+    ack!
+  end
+end
+
+

data/lib/sneakers.rb

@@ -0,0 +1,83 @@
+require "sneakers/version"
+require 'thread/pool'
+require 'bunny'
+require 'logger'
+
+module Sneakers
+  module Handlers
+  end
+  module Concerns
+  end
+end
+
+require 'sneakers/configuration'
+require 'sneakers/support/production_formatter'
+require 'sneakers/concerns/logging'
+require 'sneakers/concerns/metrics'
+require 'sneakers/handlers/oneshot'
+require 'sneakers/worker'
+require 'sneakers/publisher'
+
+module Sneakers
+  extend self
+
+  CONFIG = Configuration.new
+
+  def configure(opts={})
+    # worker > userland > defaults
+    CONFIG.merge!(opts)
+
+    setup_general_logger!
+    setup_worker_concerns!
+    setup_general_publisher!
+    @configured = true
+  end
+
+  def clear!
+    CONFIG.clear
+    @logger = nil
+    @publisher = nil
+    @configured = false
+  end
+
+  def daemonize!(loglevel=Logger::INFO)
+    CONFIG[:log] = 'sneakers.log'
+    CONFIG[:daemonize] = true
+    setup_general_logger!
+    logger.level = loglevel
+  end
+
+  def logger
+    @logger
+  end
+
+  def publish(msg, routing)
+    @publisher.publish(msg, routing)
+  end
+
+  def configured?
+    @configured
+  end
+
+  private
+
+  def setup_general_logger!
+    if [:info, :debug, :error, :warn].all?{ |meth| CONFIG[:log].respond_to?(meth) }
+      @logger = CONFIG[:log]
+    else
+      @logger = Logger.new(CONFIG[:log])
+      @logger.formatter = Sneakers::Support::ProductionFormatter
+    end
+  end
+
+  def setup_worker_concerns!
+    Worker.configure_logger(Sneakers::logger)
+    Worker.configure_metrics(CONFIG[:metrics])
+    CONFIG[:handler] ||= Sneakers::Handlers::Oneshot
+  end
+
+  def setup_general_publisher!
+    @publisher = Sneakers::Publisher.new
+  end
+end
+

data/lib/sneakers/cli.rb

@@ -0,0 +1,115 @@
+require 'thor'
+require 'sneakers/runner'
+
+
+#
+# $ sneakers run TitleWorker,FooWorker
+# $ sneakers stop
+# $ sneakers recycle
+# $ sneakers reload
+# $ sneakers init
+#
+#
+module Sneakers
+  class CLI < Thor
+
+    SNEAKERS=<<-EOF
+
+        __
+    ,--'  >  Sneakers
+    `=====
+
+    EOF
+
+    BANNER = SNEAKERS
+
+    method_option :debug
+    method_option :daemonize
+    method_option :require
+
+    desc "work FirstWorker,SecondWorker ... ,NthWorker", "Run workers"
+    def work(workers)
+      opts = {
+        :daemonize => !!options[:daemonize]
+      }
+
+      opts[:log] =  opts[:daemonize] ? 'sneakers.log' : STDOUT
+
+      if opts[:daemonize]
+        puts "*** DEPRACATED: self-daemonization '--daemonize' is considered a bad practice, which is why this feature will be removed in future versions. Please run Sneakers in front, and use things like upstart, systemd, or supervisor to manage it as a daemon."
+      end
+
+
+      Sneakers.configure(opts)
+
+      require_boot File.expand_path(options[:require]) if options[:require]
+
+      workers, missing_workers = Sneakers::Utils.parse_workers(workers)
+
+      unless missing_workers.empty?
+        say "Missing workers: #{missing_workers.join(', ')}" if missing_workers
+        say "Did you `require` properly?"
+        return
+      end
+
+      if workers.empty?
+        say <<-EOF
+        Error: No workers found.
+        Please require your worker classes before specifying in CLI
+
+          $ sneakers run FooWorker
+                         ^- require this in your code
+
+        EOF
+        return
+      end
+
+      r = Sneakers::Runner.new(workers)
+
+      pid = Sneakers::CONFIG[:pid_path]
+
+      say SNEAKERS
+      say "Workers ....: #{em workers.join(', ')}"
+      say "Log ........: #{em (Sneakers::CONFIG[:log] == STDOUT ? 'Console' : Sneakers::CONFIG[:log]) }"
+      say "PID ........: #{em pid}"
+      say ""
+      say (" "*31)+"Process control"
+      say "="*80
+      say "Stop (nicely) ..............: kill -SIGTERM `cat #{pid}`"
+      say "Stop (immediate) ...........: kill -SIGQUIT `cat #{pid}`"
+      say "Restart (nicely) ...........: kill -SIGUSR1 `cat #{pid}`"
+      say "Restart (immediate) ........: kill -SIGHUP `cat #{pid}`"
+      say "Reconfigure ................: kill -SIGUSR2 `cat #{pid}`"
+      say "Scale workers ..............: reconfigure, then restart"
+      say "="*80
+      say ""
+
+      if options[:debug]
+        say "==== configuration ==="
+        say Sneakers::CONFIG.inspect
+        say "======================"
+      end
+
+      r.run
+    end
+
+
+  private
+    def require_boot(file)
+      load file
+    end
+
+    def em(text)
+      shell.set_color(text, nil, true)
+    end
+
+    def ok(detail=nil)
+      text = detail ? "OK, #{detail}." : "OK."
+      say text, :green
+    end
+
+    def error(detail)
+      say detail, :red
+    end
+  end
+end

data/lib/sneakers/concerns/logging.rb

@@ -0,0 +1,34 @@
+module Sneakers
+  module Concerns
+    module Logging
+      def self.included(base)
+        base.extend ClassMethods
+        base.send :define_method, :logger do
+          base.logger
+        end
+      end
+
+      module ClassMethods
+        def logger
+          @logger
+        end
+
+        def logger=(logger)
+          @logger = logger
+        end
+
+        def configure_logger(log=nil)
+          if log
+            @logger = log
+          else
+            @logger = Logger.new(STDOUT)
+            @logger.level = Logger::INFO
+            @logger.formatter = Sneakers::Support::ProductionFormatter
+          end
+        end
+      end
+    end
+  end
+end
+
+

data/lib/sneakers/concerns/metrics.rb

@@ -0,0 +1,34 @@
+require 'sneakers/metrics/null_metrics'
+
+module Sneakers
+  module Concerns
+    module Metrics
+      def self.included(base)
+        base.extend ClassMethods
+        base.send :define_method, :metrics do
+          base.metrics
+        end
+      end
+
+      module ClassMethods
+        def metrics
+          @metrics
+        end
+
+        def metrics=(metrics)
+          @metrics = metrics
+        end
+
+        def configure_metrics(metrics=nil)
+          if metrics
+            @metrics = metrics
+          else
+            @metrics = Sneakers::Metrics::NullMetrics.new
+          end
+        end
+      end
+    end
+  end
+end
+
+

data/lib/sneakers/configuration.rb

@@ -0,0 +1,59 @@
+require 'forwardable'
+
+module Sneakers
+  class Configuration
+
+    extend Forwardable
+    def_delegators :@hash, :to_hash, :[], :[]=, :==, :fetch, :delete
+
+    DEFAULTS = {
+      # runner
+      :runner_config_file => nil,
+      :metrics            => nil,
+      :daemonize          => false,
+      :start_worker_delay => 0.2,
+      :workers            => 4,
+      :log                => STDOUT,
+      :pid_path           => 'sneakers.pid',
+
+      # workers
+      :timeout_job_after  => 5,
+      :prefetch           => 10,
+      :threads            => 10,
+      :durable            => true,
+      :ack                => true,
+      :heartbeat          => 2,
+      :exchange           => 'sneakers',
+      :exchange_type      => :direct,
+      :hooks              => {}
+    }.freeze
+
+
+    def initialize
+      clear
+    end
+
+    def clear
+      @hash = DEFAULTS.dup
+      @hash[:amqp]  = ENV.fetch('RABBITMQ_URL', 'amqp://guest:guest@localhost:5672')
+      @hash[:vhost] = AMQ::Settings.parse_amqp_url(@hash[:amqp]).fetch(:vhost, '/')
+    end
+
+    def merge!(hash)
+      # parse vhost from amqp if vhost is not specified explicitly
+      if hash[:vhost].nil? && !hash[:amqp].nil?
+        hash = hash.dup
+        hash[:vhost] = AMQ::Settings.parse_amqp_url(hash[:amqp]).fetch(:vhost, '/')
+      end
+
+      @hash.merge!(hash)
+    end
+
+    def merge(hash)
+      instance = self.class.new
+      instance.merge! to_hash
+      instance.merge! hash
+      instance
+    end
+  end
+end