sneakers 0.1.1.pre → 1.0.0

data/lib/sneakers/worker.rb

@@ -4,17 +4,17 @@ require 'timeout'
 
 module Sneakers
   module Worker
-    attr_reader :queue, :id
+    attr_reader :queue, :id, :opts
 
     # For now, a worker is hardly dependant on these concerns
     # (because it uses methods from them directly.)
     include Concerns::Logging
     include Concerns::Metrics
 
-    def initialize(queue=nil, pool=nil, opts=nil)
-      opts = self.class.queue_opts
+    def initialize(queue = nil, pool = nil, opts = {})
+      opts = opts.merge(self.class.queue_opts || {})
       queue_name = self.class.queue_name
-      opts = Sneakers::Config.merge(opts)
+      opts = Sneakers::CONFIG.merge(opts)
 
       @should_ack =  opts[:ack]
       @timeout_after = opts[:timeout_job_after]
@@ -34,12 +34,14 @@ module Sneakers
     def reject!; :reject; end
     def requeue!; :requeue; end
 
-    def publish(msg, routing)
-      return unless routing[:to_queue]
-      @queue.exchange.publish(msg, :routing_key => routing[:to_queue])
+    def publish(msg, opts)
+      to_queue = opts.delete(:to_queue)
+      opts[:routing_key] ||= to_queue
+      return unless opts[:routing_key]
+      @queue.exchange.publish(msg, opts)
     end
 
-    def do_work(hdr, props, msg, handler)
+    def do_work(delivery_info, metadata, msg, handler)
       worker_trace "Working off: #{msg}"
 
       @pool.process do
@@ -51,7 +53,7 @@ module Sneakers
           Timeout.timeout(@timeout_after) do
             metrics.timing("work.#{self.class.name}.time") do
               if @call_with_params
-                res = work_with_params(msg, hdr, props)
+                res = work_with_params(msg, delivery_info, metadata)
               else
                 res = work(msg)
               end
@@ -59,29 +61,30 @@ module Sneakers
           end
         rescue Timeout::Error
           res = :timeout
-          logger.error("timeout")
+          worker_error('timeout')
         rescue => ex
           res = :error
           error = ex
-          logger.error(ex)
+          worker_error('unexpected error', ex)
         end
 
         if @should_ack
+
           if res == :ack
             # note to future-self. never acknowledge multiple (multiple=true) messages under threads.
-            handler.acknowledge(hdr.delivery_tag)
+            handler.acknowledge(delivery_info, metadata, msg)
           elsif res == :timeout
-            handler.timeout(hdr.delivery_tag)
+            handler.timeout(delivery_info, metadata, msg)
           elsif res == :error
-            handler.error(hdr.delivery_tag, error)
+            handler.error(delivery_info, metadata, msg, error)
           elsif res == :reject
-            handler.reject(hdr.delivery_tag)
+            handler.reject(delivery_info, metadata, msg)
           elsif res == :requeue
-            handler.reject(hdr.delivery_tag, true)
+            handler.reject(delivery_info, metadata, msg, true)
           else
-            handler.noop(hdr.delivery_tag)
+            handler.noop(delivery_info, metadata, msg)
           end
-          metrics.increment("work.#{self.class.name}.handled.#{res || 'reject'}")
+          metrics.increment("work.#{self.class.name}.handled.#{res || 'noop'}")
         end
 
         metrics.increment("work.#{self.class.name}.ended")
@@ -100,8 +103,24 @@ module Sneakers
       worker_trace "New worker: I'm alive."
     end
 
+    # Construct a log message with some standard prefix for this worker
+    def log_msg(msg)
+      "[#{@id}][#{Thread.current}][#{@queue.name}][#{@queue.opts}] #{msg}"
+    end
+
+    # Helper to log an error message with an optional exception
+    def worker_error(msg, exception = nil)
+      s = log_msg(msg)
+      if exception
+        s += " [Exception error=#{exception.message.inspect} error_class=#{exception.class}"
+        s += " backtrace=#{exception.backtrace.take(50).join(',')}" unless exception.backtrace.nil?
+        s += "]"
+      end
+      logger.error(s)
+    end
+
     def worker_trace(msg)
-      logger.debug "[#{@id}][#{Thread.current}][#{@queue.name}][#{@queue.opts}] #{msg}"
+      logger.debug(log_msg(msg))
     end
 
     def self.included(base)

data/lib/sneakers/workergroup.rb

@@ -7,12 +7,12 @@ module Sneakers
     end
 
     def before_fork
-      fbefore = Sneakers::Config[:hooks][:before_fork]
+      fbefore = Sneakers::CONFIG[:hooks][:before_fork]
       fbefore.call if fbefore
     end
 
     def after_fork # note! this is not Serverengine#after_start, this is ours!
-      fafter = Sneakers::Config[:hooks][:after_fork]
+      fafter = Sneakers::CONFIG[:hooks][:after_fork]
       fafter.call if fafter
     end
 

data/lib/sneakers.rb

@@ -3,7 +3,6 @@ require 'thread/pool'
 require 'bunny'
 require 'logger'
 
-
 module Sneakers
   module Handlers
   end
@@ -11,6 +10,7 @@ module Sneakers
   end
 end
 
+require 'sneakers/configuration'
 require 'sneakers/support/production_formatter'
 require 'sneakers/concerns/logging'
 require 'sneakers/concerns/metrics'
@@ -19,36 +19,13 @@ require 'sneakers/worker'
 require 'sneakers/publisher'
 
 module Sneakers
+  extend self
+
+  CONFIG = Configuration.new
 
-  DEFAULTS = {
-    # runner
-    :runner_config_file => nil,
-    :metrics => nil,
-    :daemonize => false,
-    :start_worker_delay => 0.2,
-    :workers => 4,
-    :log  => STDOUT,
-    :pid_path => 'sneakers.pid',
-
-    #workers
-    :timeout_job_after => 5,
-    :prefetch => 10,
-    :threads => 10,
-    :durable => true,
-    :ack => true,
-    :heartbeat => 2,
-    :amqp => 'amqp://guest:guest@localhost:5672',
-    :vhost => '/',
-    :exchange => 'sneakers',
-    :exchange_type => :direct,
-    :hooks => {}
-  }.freeze
-
-  Config = DEFAULTS.dup
-
-  def self.configure(opts={})
+  def configure(opts={})
     # worker > userland > defaults
-    Config.merge!(opts)
+    CONFIG.merge!(opts)
 
     setup_general_logger!
     setup_worker_concerns!
@@ -56,52 +33,50 @@ module Sneakers
     @configured = true
   end
 
-  def self.clear!
-    Config.clear
-    Config.merge!(DEFAULTS.dup)
+  def clear!
+    CONFIG.clear
     @logger = nil
     @publisher = nil
     @configured = false
   end
 
-  def self.daemonize!(loglevel=Logger::INFO)
-    Config[:log] = 'sneakers.log'
-    Config[:daemonize] = true
+  def daemonize!(loglevel=Logger::INFO)
+    CONFIG[:log] = 'sneakers.log'
+    CONFIG[:daemonize] = true
     setup_general_logger!
     logger.level = loglevel
   end
 
-  def self.logger
+  def logger
     @logger
   end
 
-  def self.publish(msg, routing)
+  def publish(msg, routing)
     @publisher.publish(msg, routing)
   end
 
-  def self.configured?
+  def configured?
     @configured
   end
 
+  private
 
-private
-
-  def self.setup_general_logger!
-    if [:info, :debug, :error, :warn].all?{ |meth| Config[:log].respond_to?(meth) }
-      @logger = Config[:log]
+  def setup_general_logger!
+    if [:info, :debug, :error, :warn].all?{ |meth| CONFIG[:log].respond_to?(meth) }
+      @logger = CONFIG[:log]
     else
-      @logger = Logger.new(Config[:log])
+      @logger = Logger.new(CONFIG[:log])
       @logger.formatter = Sneakers::Support::ProductionFormatter
     end
   end
 
-  def self.setup_worker_concerns!
+  def setup_worker_concerns!
     Worker.configure_logger(Sneakers::logger)
-    Worker.configure_metrics(Config[:metrics])
-    Config[:handler] ||= Sneakers::Handlers::Oneshot
+    Worker.configure_metrics(CONFIG[:metrics])
+    CONFIG[:handler] ||= Sneakers::Handlers::Oneshot
   end
 
-  def self.setup_general_publisher!
+  def setup_general_publisher!
     @publisher = Sneakers::Publisher.new
   end
 end

data/sneakers.gemspec

@@ -4,30 +4,30 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'sneakers/version'
 
 Gem::Specification.new do |gem|
-  gem.name          = "sneakers"
+  gem.name          = 'sneakers'
   gem.version       = Sneakers::VERSION
-  gem.authors       = ["Dotan Nahum"]
-  gem.email         = ["jondotan@gmail.com"]
-  gem.description   = %q{Fast background processing framework for Ruby and RabbitMQ}
-  gem.summary       = %q{Fast background processing framework for Ruby and RabbitMQ}
-  gem.homepage      = ""
+  gem.authors       = ['Dotan Nahum']
+  gem.email         = ['jondotan@gmail.com']
+  gem.description   = %q( Fast background processing framework for Ruby and RabbitMQ )
+  gem.summary       = %q( Fast background processing framework for Ruby and RabbitMQ )
+  gem.homepage      = ''
 
-  gem.files         = `git ls-files`.split($/).reject{|f| f == 'Gemfile.lock'}
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
-  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
-  gem.require_paths = ["lib"]
-  gem.add_dependency "serverengine"
-  gem.add_dependency "bunny", "~> 1.1.3"
-  gem.add_dependency "thread"
-  gem.add_dependency "thor"
+  gem.files         = `git ls-files`.split($/).reject { |f| f == 'Gemfile.lock' }
+  gem.executables   = gem.files.grep(/^bin/).map { |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(/^(test|spec|features)\//)
+  gem.require_paths = ['lib']
+  gem.add_dependency 'serverengine'
+  gem.add_dependency 'bunny', '~> 1.1.3'
+  gem.add_dependency 'thread'
+  gem.add_dependency 'thor'
 
-  gem.add_development_dependency "rr"
-  gem.add_development_dependency "ruby-prof"
-  gem.add_development_dependency "nokogiri"
-  gem.add_development_dependency "guard-minitest"
-  gem.add_development_dependency "metric_fu"
-  gem.add_development_dependency "simplecov"
-  gem.add_development_dependency "simplecov-rcov-text"
-  gem.add_development_dependency "rake"
+  gem.add_development_dependency 'rr'
+  gem.add_development_dependency 'ruby-prof'
+  gem.add_development_dependency 'nokogiri'
+  gem.add_development_dependency 'guard-minitest'
+  gem.add_development_dependency 'metric_fu'
+  gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'simplecov-rcov-text'
+  gem.add_development_dependency 'rake'
 end
 

data/spec/sneakers/concerns/{logging.rb → logging_spec.rb}

@@ -17,7 +17,7 @@ describe Sneakers::Concerns::Logging do
       Foobar.logger.must_be_nil
       Foobar.configure_logger
       Foobar.logger.wont_be_nil
-      Foobar.logger.formatter.must_equal Sneakers::Concerns::Logging::ProductionFormatter
+      Foobar.logger.formatter.must_equal Sneakers::Support::ProductionFormatter
     end
 
     it "should supply accessible instance logger" do

data/spec/sneakers/configuration_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+
+describe Sneakers::Configuration do
+
+  it 'should assign a default value for :amqp' do
+    with_env('RABBITMQ_URL', nil) do
+      config = Sneakers::Configuration.new
+      config[:amqp].must_equal 'amqp://guest:guest@localhost:5672'
+    end
+  end
+
+  it 'should assign a default value for :vhost' do
+    with_env('RABBITMQ_URL', nil) do
+      config = Sneakers::Configuration.new
+      config[:vhost].must_equal '/'
+    end
+  end
+
+  it 'should read the value for amqp from RABBITMQ_URL' do
+    url = 'amqp://foo:bar@localhost:5672'
+    with_env('RABBITMQ_URL', url) do
+      config = Sneakers::Configuration.new
+      config[:amqp].must_equal url
+    end
+  end
+
+  it 'should read the value for vhost from RABBITMQ_URL' do
+    url = 'amqp://foo:bar@localhost:5672/foobarvhost'
+    with_env('RABBITMQ_URL', url) do
+      config = Sneakers::Configuration.new
+      config[:vhost].must_equal 'foobarvhost'
+    end
+  end
+
+  def with_env(key, value)
+    old_value = ENV[key]
+    ENV[key] = value
+    yield
+  ensure
+    ENV[key] = old_value
+  end
+end

data/spec/sneakers/publisher_spec.rb

@@ -1,53 +1,77 @@
 require 'spec_helper'
 require 'sneakers'
 
-
 describe Sneakers::Publisher do
-  describe "#publish" do
-    it "should publish a message to an exchange" do
+  describe '#publish' do
+    it 'should publish a message to an exchange' do
+      xchg = Object.new
+      mock(xchg).publish('test msg', routing_key: 'downloads')
+
+      p = Sneakers::Publisher.new
+      p.instance_variable_set(:@exchange, xchg)
+
+      mock(p).ensure_connection! {}
+      p.publish('test msg', to_queue: 'downloads')
+    end
+
+    it 'should publish with the persistence specified' do
       xchg = Object.new
-      mock(xchg).publish("test msg", :routing_key => "downloads")
+      mock(xchg).publish('test msg', routing_key: 'downloads', persistence: true)
 
       p = Sneakers::Publisher.new
-      p.exchange = xchg
+      p.instance_variable_set(:@exchange, xchg)
 
-      mock(p).ensure_connection!{}
-      p.publish("test msg", :to_queue => 'downloads')
+      mock(p).ensure_connection! {}
+      p.publish('test msg', to_queue: 'downloads', persistence: true)
     end
 
-    it "should not reconnect if already connected" do
+    it 'should publish with arbitrary metadata specified' do
       xchg = Object.new
-      mock(xchg).publish("test msg", :routing_key => "downloads")
+      mock(xchg).publish('test msg', routing_key: 'downloads', expiration: 1, headers: {foo: 'bar'})
 
       p = Sneakers::Publisher.new
-      p.exchange = xchg
-      mock(p).connected?{true}
+      p.instance_variable_set(:@exchange, xchg)
+
+      mock(p).ensure_connection! {}
+      p.publish('test msg', to_queue: 'downloads', expiration: 1, headers: {foo: 'bar'})
+    end
+
+    it 'should not reconnect if already connected' do
+      xchg = Object.new
+      mock(xchg).publish('test msg', routing_key: 'downloads')
+
+      p = Sneakers::Publisher.new
+      p.instance_variable_set(:@exchange, xchg)
+
+      mock(p).connected? { true }
       mock(p).ensure_connection!.times(0)
 
-      p.publish("test msg", :to_queue => 'downloads')
+      p.publish('test msg', to_queue: 'downloads')
     end
 
-    it "should connect to rabbitmq configured on Sneakers.configure" do
+    it 'should connect to rabbitmq configured on Sneakers.configure' do
+      logger = Logger.new('/dev/null')
       Sneakers.configure(
-        :amqp => "amqp://someuser:somepassword@somehost:5672",
-        :heartbeat => 1, :exchange => 'another_exchange',
-        :exchange_type => :topic,
-        :durable => false)
+        amqp: 'amqp://someuser:somepassword@somehost:5672',
+        heartbeat: 1, exchange: 'another_exchange',
+        exchange_type: :topic,
+        log: logger,
+        durable: false)
 
       channel = Object.new
-      mock(channel).exchange("another_exchange", :type => :topic, :durable => false) {
-        mock(Object.new).publish("test msg", :routing_key => "downloads")
-      }
+      mock(channel).exchange('another_exchange', type: :topic, durable: false) do
+        mock(Object.new).publish('test msg', routing_key: 'downloads')
+      end
 
       bunny = Object.new
       mock(bunny).start
       mock(bunny).create_channel { channel }
 
-      mock(Bunny).new("amqp://someuser:somepassword@somehost:5672", :heartbeat => 1 ) { bunny }
+      mock(Bunny).new('amqp://someuser:somepassword@somehost:5672', heartbeat: 1, vhost: '/', logger: logger) { bunny }
 
       p = Sneakers::Publisher.new
 
-      p.publish("test msg", :to_queue => 'downloads')
+      p.publish('test msg', to_queue: 'downloads')
 
     end
   end

data/spec/sneakers/queue_spec.rb

@@ -26,6 +26,8 @@ describe Sneakers::Queue do
       @mkchan = Object.new
       @mkex = Object.new
       @mkqueue = Object.new
+      @mkqueue_nondurable = Object.new
+      @mkworker = Object.new
 
       mock(@mkbunny).start {}
       mock(@mkbunny).create_channel{ @mkchan }
@@ -33,19 +35,22 @@ describe Sneakers::Queue do
 
       mock(@mkchan).prefetch(25)
       mock(@mkchan).exchange("sneakers", :type => :direct, :durable => true){ @mkex }
-      mock(@mkchan).queue("downloads", :durable => true){ @mkqueue }
+
+      stub(@mkworker).opts { { :exchange => 'test-exchange' } }
     end
 
     it "should setup a bunny queue according to configuration values" do
+      mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
       q = Sneakers::Queue.new("downloads", queue_vars)
 
       mock(@mkqueue).bind(@mkex, :routing_key => "downloads")
       mock(@mkqueue).subscribe(:block => false, :ack => true)
 
-      q.subscribe(Object.new)
+      q.subscribe(@mkworker)
     end
 
     it "supports multiple routing_keys" do
+      mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
       q = Sneakers::Queue.new("downloads",
                               queue_vars.merge(:routing_key => ["alpha", "beta"]))
 
@@ -53,10 +58,33 @@ describe Sneakers::Queue do
       mock(@mkqueue).bind(@mkex, :routing_key => "beta")
       mock(@mkqueue).subscribe(:block => false, :ack => true)
 
-      q.subscribe(Object.new)
+      q.subscribe(@mkworker)
     end
-  end
 
+    it "will use whatever handler the worker specifies" do
+      mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
+      @handler = Object.new
+      worker_opts = { :handler => @handler }
+      stub(@mkworker).opts { worker_opts }
+      mock(@handler).new(@mkchan, @mkqueue, worker_opts).once
+
+      stub(@mkqueue).bind
+      stub(@mkqueue).subscribe
+      q = Sneakers::Queue.new("downloads", queue_vars)
+      q.subscribe(@mkworker)
+    end
 
+    it "creates a non-durable queue if :queue_durable => false" do
+      mock(@mkchan).queue("test_nondurable", :durable => false) { @mkqueue_nondurable }
+      queue_vars[:queue_durable] = false
+      q = Sneakers::Queue.new("test_nondurable", queue_vars)
+
+      mock(@mkqueue_nondurable).bind(@mkex, :routing_key => "test_nondurable")
+      mock(@mkqueue_nondurable).subscribe(:block => false, :ack => true)
+
+      q.subscribe(@mkworker)
+      myqueue = q.instance_variable_get(:@queue)
+    end
+  end
 end
 

data/spec/sneakers/sneakers_spec.rb

@@ -17,20 +17,21 @@ describe Sneakers do
 
   describe 'self' do
     it 'should have defaults set up' do
-      Sneakers::Config[:log].must_equal(STDOUT)
+      Sneakers::CONFIG[:log].must_equal(STDOUT)
     end
 
     it 'should configure itself' do
       Sneakers.configure
       Sneakers.logger.wont_be_nil
+      Sneakers.configured?.must_equal(true)
     end
   end
 
   describe '.daemonize!' do
     it 'should set a logger to a default info level and not daemonize' do
       Sneakers.daemonize!
-      Sneakers::Config[:log].must_equal('sneakers.log')
-      Sneakers::Config[:daemonize].must_equal(true)
+      Sneakers::CONFIG[:log].must_equal('sneakers.log')
+      Sneakers::CONFIG[:daemonize].must_equal(true)
       Sneakers.logger.level.must_equal(Logger::INFO)
     end
 
@@ -43,11 +44,11 @@ describe Sneakers do
 
   describe '.clear!' do
     it 'must reset dirty configuration to default' do
-      Sneakers::Config[:log].must_equal(STDOUT)
+      Sneakers::CONFIG[:log].must_equal(STDOUT)
       Sneakers.configure(:log => 'foobar.log')
-      Sneakers::Config[:log].must_equal('foobar.log')
+      Sneakers::CONFIG[:log].must_equal('foobar.log')
       Sneakers.clear!
-      Sneakers::Config[:log].must_equal(STDOUT)
+      Sneakers::CONFIG[:log].must_equal(STDOUT)
     end
   end