sneakers 0.1.1.pre → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,17 +4,17 @@ require 'timeout'
4
4
 
5
5
  module Sneakers
6
6
  module Worker
7
- attr_reader :queue, :id
7
+ attr_reader :queue, :id, :opts
8
8
 
9
9
  # For now, a worker is hardly dependant on these concerns
10
10
  # (because it uses methods from them directly.)
11
11
  include Concerns::Logging
12
12
  include Concerns::Metrics
13
13
 
14
- def initialize(queue=nil, pool=nil, opts=nil)
15
- opts = self.class.queue_opts
14
+ def initialize(queue = nil, pool = nil, opts = {})
15
+ opts = opts.merge(self.class.queue_opts || {})
16
16
  queue_name = self.class.queue_name
17
- opts = Sneakers::Config.merge(opts)
17
+ opts = Sneakers::CONFIG.merge(opts)
18
18
 
19
19
  @should_ack = opts[:ack]
20
20
  @timeout_after = opts[:timeout_job_after]
@@ -34,12 +34,14 @@ module Sneakers
34
34
  def reject!; :reject; end
35
35
  def requeue!; :requeue; end
36
36
 
37
- def publish(msg, routing)
38
- return unless routing[:to_queue]
39
- @queue.exchange.publish(msg, :routing_key => routing[:to_queue])
37
+ def publish(msg, opts)
38
+ to_queue = opts.delete(:to_queue)
39
+ opts[:routing_key] ||= to_queue
40
+ return unless opts[:routing_key]
41
+ @queue.exchange.publish(msg, opts)
40
42
  end
41
43
 
42
- def do_work(hdr, props, msg, handler)
44
+ def do_work(delivery_info, metadata, msg, handler)
43
45
  worker_trace "Working off: #{msg}"
44
46
 
45
47
  @pool.process do
@@ -51,7 +53,7 @@ module Sneakers
51
53
  Timeout.timeout(@timeout_after) do
52
54
  metrics.timing("work.#{self.class.name}.time") do
53
55
  if @call_with_params
54
- res = work_with_params(msg, hdr, props)
56
+ res = work_with_params(msg, delivery_info, metadata)
55
57
  else
56
58
  res = work(msg)
57
59
  end
@@ -59,29 +61,30 @@ module Sneakers
59
61
  end
60
62
  rescue Timeout::Error
61
63
  res = :timeout
62
- logger.error("timeout")
64
+ worker_error('timeout')
63
65
  rescue => ex
64
66
  res = :error
65
67
  error = ex
66
- logger.error(ex)
68
+ worker_error('unexpected error', ex)
67
69
  end
68
70
 
69
71
  if @should_ack
72
+
70
73
  if res == :ack
71
74
  # note to future-self. never acknowledge multiple (multiple=true) messages under threads.
72
- handler.acknowledge(hdr.delivery_tag)
75
+ handler.acknowledge(delivery_info, metadata, msg)
73
76
  elsif res == :timeout
74
- handler.timeout(hdr.delivery_tag)
77
+ handler.timeout(delivery_info, metadata, msg)
75
78
  elsif res == :error
76
- handler.error(hdr.delivery_tag, error)
79
+ handler.error(delivery_info, metadata, msg, error)
77
80
  elsif res == :reject
78
- handler.reject(hdr.delivery_tag)
81
+ handler.reject(delivery_info, metadata, msg)
79
82
  elsif res == :requeue
80
- handler.reject(hdr.delivery_tag, true)
83
+ handler.reject(delivery_info, metadata, msg, true)
81
84
  else
82
- handler.noop(hdr.delivery_tag)
85
+ handler.noop(delivery_info, metadata, msg)
83
86
  end
84
- metrics.increment("work.#{self.class.name}.handled.#{res || 'reject'}")
87
+ metrics.increment("work.#{self.class.name}.handled.#{res || 'noop'}")
85
88
  end
86
89
 
87
90
  metrics.increment("work.#{self.class.name}.ended")
@@ -100,8 +103,24 @@ module Sneakers
100
103
  worker_trace "New worker: I'm alive."
101
104
  end
102
105
 
106
+ # Construct a log message with some standard prefix for this worker
107
+ def log_msg(msg)
108
+ "[#{@id}][#{Thread.current}][#{@queue.name}][#{@queue.opts}] #{msg}"
109
+ end
110
+
111
+ # Helper to log an error message with an optional exception
112
+ def worker_error(msg, exception = nil)
113
+ s = log_msg(msg)
114
+ if exception
115
+ s += " [Exception error=#{exception.message.inspect} error_class=#{exception.class}"
116
+ s += " backtrace=#{exception.backtrace.take(50).join(',')}" unless exception.backtrace.nil?
117
+ s += "]"
118
+ end
119
+ logger.error(s)
120
+ end
121
+
103
122
  def worker_trace(msg)
104
- logger.debug "[#{@id}][#{Thread.current}][#{@queue.name}][#{@queue.opts}] #{msg}"
123
+ logger.debug(log_msg(msg))
105
124
  end
106
125
 
107
126
  def self.included(base)
@@ -7,12 +7,12 @@ module Sneakers
7
7
  end
8
8
 
9
9
  def before_fork
10
- fbefore = Sneakers::Config[:hooks][:before_fork]
10
+ fbefore = Sneakers::CONFIG[:hooks][:before_fork]
11
11
  fbefore.call if fbefore
12
12
  end
13
13
 
14
14
  def after_fork # note! this is not Serverengine#after_start, this is ours!
15
- fafter = Sneakers::Config[:hooks][:after_fork]
15
+ fafter = Sneakers::CONFIG[:hooks][:after_fork]
16
16
  fafter.call if fafter
17
17
  end
18
18
 
data/lib/sneakers.rb CHANGED
@@ -3,7 +3,6 @@ require 'thread/pool'
3
3
  require 'bunny'
4
4
  require 'logger'
5
5
 
6
-
7
6
  module Sneakers
8
7
  module Handlers
9
8
  end
@@ -11,6 +10,7 @@ module Sneakers
11
10
  end
12
11
  end
13
12
 
13
+ require 'sneakers/configuration'
14
14
  require 'sneakers/support/production_formatter'
15
15
  require 'sneakers/concerns/logging'
16
16
  require 'sneakers/concerns/metrics'
@@ -19,36 +19,13 @@ require 'sneakers/worker'
19
19
  require 'sneakers/publisher'
20
20
 
21
21
  module Sneakers
22
+ extend self
23
+
24
+ CONFIG = Configuration.new
22
25
 
23
- DEFAULTS = {
24
- # runner
25
- :runner_config_file => nil,
26
- :metrics => nil,
27
- :daemonize => false,
28
- :start_worker_delay => 0.2,
29
- :workers => 4,
30
- :log => STDOUT,
31
- :pid_path => 'sneakers.pid',
32
-
33
- #workers
34
- :timeout_job_after => 5,
35
- :prefetch => 10,
36
- :threads => 10,
37
- :durable => true,
38
- :ack => true,
39
- :heartbeat => 2,
40
- :amqp => 'amqp://guest:guest@localhost:5672',
41
- :vhost => '/',
42
- :exchange => 'sneakers',
43
- :exchange_type => :direct,
44
- :hooks => {}
45
- }.freeze
46
-
47
- Config = DEFAULTS.dup
48
-
49
- def self.configure(opts={})
26
+ def configure(opts={})
50
27
  # worker > userland > defaults
51
- Config.merge!(opts)
28
+ CONFIG.merge!(opts)
52
29
 
53
30
  setup_general_logger!
54
31
  setup_worker_concerns!
@@ -56,52 +33,50 @@ module Sneakers
56
33
  @configured = true
57
34
  end
58
35
 
59
- def self.clear!
60
- Config.clear
61
- Config.merge!(DEFAULTS.dup)
36
+ def clear!
37
+ CONFIG.clear
62
38
  @logger = nil
63
39
  @publisher = nil
64
40
  @configured = false
65
41
  end
66
42
 
67
- def self.daemonize!(loglevel=Logger::INFO)
68
- Config[:log] = 'sneakers.log'
69
- Config[:daemonize] = true
43
+ def daemonize!(loglevel=Logger::INFO)
44
+ CONFIG[:log] = 'sneakers.log'
45
+ CONFIG[:daemonize] = true
70
46
  setup_general_logger!
71
47
  logger.level = loglevel
72
48
  end
73
49
 
74
- def self.logger
50
+ def logger
75
51
  @logger
76
52
  end
77
53
 
78
- def self.publish(msg, routing)
54
+ def publish(msg, routing)
79
55
  @publisher.publish(msg, routing)
80
56
  end
81
57
 
82
- def self.configured?
58
+ def configured?
83
59
  @configured
84
60
  end
85
61
 
62
+ private
86
63
 
87
- private
88
-
89
- def self.setup_general_logger!
90
- if [:info, :debug, :error, :warn].all?{ |meth| Config[:log].respond_to?(meth) }
91
- @logger = Config[:log]
64
+ def setup_general_logger!
65
+ if [:info, :debug, :error, :warn].all?{ |meth| CONFIG[:log].respond_to?(meth) }
66
+ @logger = CONFIG[:log]
92
67
  else
93
- @logger = Logger.new(Config[:log])
68
+ @logger = Logger.new(CONFIG[:log])
94
69
  @logger.formatter = Sneakers::Support::ProductionFormatter
95
70
  end
96
71
  end
97
72
 
98
- def self.setup_worker_concerns!
73
+ def setup_worker_concerns!
99
74
  Worker.configure_logger(Sneakers::logger)
100
- Worker.configure_metrics(Config[:metrics])
101
- Config[:handler] ||= Sneakers::Handlers::Oneshot
75
+ Worker.configure_metrics(CONFIG[:metrics])
76
+ CONFIG[:handler] ||= Sneakers::Handlers::Oneshot
102
77
  end
103
78
 
104
- def self.setup_general_publisher!
79
+ def setup_general_publisher!
105
80
  @publisher = Sneakers::Publisher.new
106
81
  end
107
82
  end
data/sneakers.gemspec CHANGED
@@ -4,30 +4,30 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
  require 'sneakers/version'
5
5
 
6
6
  Gem::Specification.new do |gem|
7
- gem.name = "sneakers"
7
+ gem.name = 'sneakers'
8
8
  gem.version = Sneakers::VERSION
9
- gem.authors = ["Dotan Nahum"]
10
- gem.email = ["jondotan@gmail.com"]
11
- gem.description = %q{Fast background processing framework for Ruby and RabbitMQ}
12
- gem.summary = %q{Fast background processing framework for Ruby and RabbitMQ}
13
- gem.homepage = ""
9
+ gem.authors = ['Dotan Nahum']
10
+ gem.email = ['jondotan@gmail.com']
11
+ gem.description = %q( Fast background processing framework for Ruby and RabbitMQ )
12
+ gem.summary = %q( Fast background processing framework for Ruby and RabbitMQ )
13
+ gem.homepage = ''
14
14
 
15
- gem.files = `git ls-files`.split($/).reject{|f| f == 'Gemfile.lock'}
16
- gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
17
- gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
18
- gem.require_paths = ["lib"]
19
- gem.add_dependency "serverengine"
20
- gem.add_dependency "bunny", "~> 1.1.3"
21
- gem.add_dependency "thread"
22
- gem.add_dependency "thor"
15
+ gem.files = `git ls-files`.split($/).reject { |f| f == 'Gemfile.lock' }
16
+ gem.executables = gem.files.grep(/^bin/).map { |f| File.basename(f) }
17
+ gem.test_files = gem.files.grep(/^(test|spec|features)\//)
18
+ gem.require_paths = ['lib']
19
+ gem.add_dependency 'serverengine'
20
+ gem.add_dependency 'bunny', '~> 1.1.3'
21
+ gem.add_dependency 'thread'
22
+ gem.add_dependency 'thor'
23
23
 
24
- gem.add_development_dependency "rr"
25
- gem.add_development_dependency "ruby-prof"
26
- gem.add_development_dependency "nokogiri"
27
- gem.add_development_dependency "guard-minitest"
28
- gem.add_development_dependency "metric_fu"
29
- gem.add_development_dependency "simplecov"
30
- gem.add_development_dependency "simplecov-rcov-text"
31
- gem.add_development_dependency "rake"
24
+ gem.add_development_dependency 'rr'
25
+ gem.add_development_dependency 'ruby-prof'
26
+ gem.add_development_dependency 'nokogiri'
27
+ gem.add_development_dependency 'guard-minitest'
28
+ gem.add_development_dependency 'metric_fu'
29
+ gem.add_development_dependency 'simplecov'
30
+ gem.add_development_dependency 'simplecov-rcov-text'
31
+ gem.add_development_dependency 'rake'
32
32
  end
33
33
 
@@ -17,7 +17,7 @@ describe Sneakers::Concerns::Logging do
17
17
  Foobar.logger.must_be_nil
18
18
  Foobar.configure_logger
19
19
  Foobar.logger.wont_be_nil
20
- Foobar.logger.formatter.must_equal Sneakers::Concerns::Logging::ProductionFormatter
20
+ Foobar.logger.formatter.must_equal Sneakers::Support::ProductionFormatter
21
21
  end
22
22
 
23
23
  it "should supply accessible instance logger" do
@@ -0,0 +1,42 @@
1
+ require 'spec_helper'
2
+
3
+ describe Sneakers::Configuration do
4
+
5
+ it 'should assign a default value for :amqp' do
6
+ with_env('RABBITMQ_URL', nil) do
7
+ config = Sneakers::Configuration.new
8
+ config[:amqp].must_equal 'amqp://guest:guest@localhost:5672'
9
+ end
10
+ end
11
+
12
+ it 'should assign a default value for :vhost' do
13
+ with_env('RABBITMQ_URL', nil) do
14
+ config = Sneakers::Configuration.new
15
+ config[:vhost].must_equal '/'
16
+ end
17
+ end
18
+
19
+ it 'should read the value for amqp from RABBITMQ_URL' do
20
+ url = 'amqp://foo:bar@localhost:5672'
21
+ with_env('RABBITMQ_URL', url) do
22
+ config = Sneakers::Configuration.new
23
+ config[:amqp].must_equal url
24
+ end
25
+ end
26
+
27
+ it 'should read the value for vhost from RABBITMQ_URL' do
28
+ url = 'amqp://foo:bar@localhost:5672/foobarvhost'
29
+ with_env('RABBITMQ_URL', url) do
30
+ config = Sneakers::Configuration.new
31
+ config[:vhost].must_equal 'foobarvhost'
32
+ end
33
+ end
34
+
35
+ def with_env(key, value)
36
+ old_value = ENV[key]
37
+ ENV[key] = value
38
+ yield
39
+ ensure
40
+ ENV[key] = old_value
41
+ end
42
+ end
@@ -1,53 +1,77 @@
1
1
  require 'spec_helper'
2
2
  require 'sneakers'
3
3
 
4
-
5
4
  describe Sneakers::Publisher do
6
- describe "#publish" do
7
- it "should publish a message to an exchange" do
5
+ describe '#publish' do
6
+ it 'should publish a message to an exchange' do
7
+ xchg = Object.new
8
+ mock(xchg).publish('test msg', routing_key: 'downloads')
9
+
10
+ p = Sneakers::Publisher.new
11
+ p.instance_variable_set(:@exchange, xchg)
12
+
13
+ mock(p).ensure_connection! {}
14
+ p.publish('test msg', to_queue: 'downloads')
15
+ end
16
+
17
+ it 'should publish with the persistence specified' do
8
18
  xchg = Object.new
9
- mock(xchg).publish("test msg", :routing_key => "downloads")
19
+ mock(xchg).publish('test msg', routing_key: 'downloads', persistence: true)
10
20
 
11
21
  p = Sneakers::Publisher.new
12
- p.exchange = xchg
22
+ p.instance_variable_set(:@exchange, xchg)
13
23
 
14
- mock(p).ensure_connection!{}
15
- p.publish("test msg", :to_queue => 'downloads')
24
+ mock(p).ensure_connection! {}
25
+ p.publish('test msg', to_queue: 'downloads', persistence: true)
16
26
  end
17
27
 
18
- it "should not reconnect if already connected" do
28
+ it 'should publish with arbitrary metadata specified' do
19
29
  xchg = Object.new
20
- mock(xchg).publish("test msg", :routing_key => "downloads")
30
+ mock(xchg).publish('test msg', routing_key: 'downloads', expiration: 1, headers: {foo: 'bar'})
21
31
 
22
32
  p = Sneakers::Publisher.new
23
- p.exchange = xchg
24
- mock(p).connected?{true}
33
+ p.instance_variable_set(:@exchange, xchg)
34
+
35
+ mock(p).ensure_connection! {}
36
+ p.publish('test msg', to_queue: 'downloads', expiration: 1, headers: {foo: 'bar'})
37
+ end
38
+
39
+ it 'should not reconnect if already connected' do
40
+ xchg = Object.new
41
+ mock(xchg).publish('test msg', routing_key: 'downloads')
42
+
43
+ p = Sneakers::Publisher.new
44
+ p.instance_variable_set(:@exchange, xchg)
45
+
46
+ mock(p).connected? { true }
25
47
  mock(p).ensure_connection!.times(0)
26
48
 
27
- p.publish("test msg", :to_queue => 'downloads')
49
+ p.publish('test msg', to_queue: 'downloads')
28
50
  end
29
51
 
30
- it "should connect to rabbitmq configured on Sneakers.configure" do
52
+ it 'should connect to rabbitmq configured on Sneakers.configure' do
53
+ logger = Logger.new('/dev/null')
31
54
  Sneakers.configure(
32
- :amqp => "amqp://someuser:somepassword@somehost:5672",
33
- :heartbeat => 1, :exchange => 'another_exchange',
34
- :exchange_type => :topic,
35
- :durable => false)
55
+ amqp: 'amqp://someuser:somepassword@somehost:5672',
56
+ heartbeat: 1, exchange: 'another_exchange',
57
+ exchange_type: :topic,
58
+ log: logger,
59
+ durable: false)
36
60
 
37
61
  channel = Object.new
38
- mock(channel).exchange("another_exchange", :type => :topic, :durable => false) {
39
- mock(Object.new).publish("test msg", :routing_key => "downloads")
40
- }
62
+ mock(channel).exchange('another_exchange', type: :topic, durable: false) do
63
+ mock(Object.new).publish('test msg', routing_key: 'downloads')
64
+ end
41
65
 
42
66
  bunny = Object.new
43
67
  mock(bunny).start
44
68
  mock(bunny).create_channel { channel }
45
69
 
46
- mock(Bunny).new("amqp://someuser:somepassword@somehost:5672", :heartbeat => 1 ) { bunny }
70
+ mock(Bunny).new('amqp://someuser:somepassword@somehost:5672', heartbeat: 1, vhost: '/', logger: logger) { bunny }
47
71
 
48
72
  p = Sneakers::Publisher.new
49
73
 
50
- p.publish("test msg", :to_queue => 'downloads')
74
+ p.publish('test msg', to_queue: 'downloads')
51
75
 
52
76
  end
53
77
  end
@@ -26,6 +26,8 @@ describe Sneakers::Queue do
26
26
  @mkchan = Object.new
27
27
  @mkex = Object.new
28
28
  @mkqueue = Object.new
29
+ @mkqueue_nondurable = Object.new
30
+ @mkworker = Object.new
29
31
 
30
32
  mock(@mkbunny).start {}
31
33
  mock(@mkbunny).create_channel{ @mkchan }
@@ -33,19 +35,22 @@ describe Sneakers::Queue do
33
35
 
34
36
  mock(@mkchan).prefetch(25)
35
37
  mock(@mkchan).exchange("sneakers", :type => :direct, :durable => true){ @mkex }
36
- mock(@mkchan).queue("downloads", :durable => true){ @mkqueue }
38
+
39
+ stub(@mkworker).opts { { :exchange => 'test-exchange' } }
37
40
  end
38
41
 
39
42
  it "should setup a bunny queue according to configuration values" do
43
+ mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
40
44
  q = Sneakers::Queue.new("downloads", queue_vars)
41
45
 
42
46
  mock(@mkqueue).bind(@mkex, :routing_key => "downloads")
43
47
  mock(@mkqueue).subscribe(:block => false, :ack => true)
44
48
 
45
- q.subscribe(Object.new)
49
+ q.subscribe(@mkworker)
46
50
  end
47
51
 
48
52
  it "supports multiple routing_keys" do
53
+ mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
49
54
  q = Sneakers::Queue.new("downloads",
50
55
  queue_vars.merge(:routing_key => ["alpha", "beta"]))
51
56
 
@@ -53,10 +58,33 @@ describe Sneakers::Queue do
53
58
  mock(@mkqueue).bind(@mkex, :routing_key => "beta")
54
59
  mock(@mkqueue).subscribe(:block => false, :ack => true)
55
60
 
56
- q.subscribe(Object.new)
61
+ q.subscribe(@mkworker)
57
62
  end
58
- end
59
63
 
64
+ it "will use whatever handler the worker specifies" do
65
+ mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
66
+ @handler = Object.new
67
+ worker_opts = { :handler => @handler }
68
+ stub(@mkworker).opts { worker_opts }
69
+ mock(@handler).new(@mkchan, @mkqueue, worker_opts).once
70
+
71
+ stub(@mkqueue).bind
72
+ stub(@mkqueue).subscribe
73
+ q = Sneakers::Queue.new("downloads", queue_vars)
74
+ q.subscribe(@mkworker)
75
+ end
60
76
 
77
+ it "creates a non-durable queue if :queue_durable => false" do
78
+ mock(@mkchan).queue("test_nondurable", :durable => false) { @mkqueue_nondurable }
79
+ queue_vars[:queue_durable] = false
80
+ q = Sneakers::Queue.new("test_nondurable", queue_vars)
81
+
82
+ mock(@mkqueue_nondurable).bind(@mkex, :routing_key => "test_nondurable")
83
+ mock(@mkqueue_nondurable).subscribe(:block => false, :ack => true)
84
+
85
+ q.subscribe(@mkworker)
86
+ myqueue = q.instance_variable_get(:@queue)
87
+ end
88
+ end
61
89
  end
62
90
 
@@ -17,20 +17,21 @@ describe Sneakers do
17
17
 
18
18
  describe 'self' do
19
19
  it 'should have defaults set up' do
20
- Sneakers::Config[:log].must_equal(STDOUT)
20
+ Sneakers::CONFIG[:log].must_equal(STDOUT)
21
21
  end
22
22
 
23
23
  it 'should configure itself' do
24
24
  Sneakers.configure
25
25
  Sneakers.logger.wont_be_nil
26
+ Sneakers.configured?.must_equal(true)
26
27
  end
27
28
  end
28
29
 
29
30
  describe '.daemonize!' do
30
31
  it 'should set a logger to a default info level and not daemonize' do
31
32
  Sneakers.daemonize!
32
- Sneakers::Config[:log].must_equal('sneakers.log')
33
- Sneakers::Config[:daemonize].must_equal(true)
33
+ Sneakers::CONFIG[:log].must_equal('sneakers.log')
34
+ Sneakers::CONFIG[:daemonize].must_equal(true)
34
35
  Sneakers.logger.level.must_equal(Logger::INFO)
35
36
  end
36
37
 
@@ -43,11 +44,11 @@ describe Sneakers do
43
44
 
44
45
  describe '.clear!' do
45
46
  it 'must reset dirty configuration to default' do
46
- Sneakers::Config[:log].must_equal(STDOUT)
47
+ Sneakers::CONFIG[:log].must_equal(STDOUT)
47
48
  Sneakers.configure(:log => 'foobar.log')
48
- Sneakers::Config[:log].must_equal('foobar.log')
49
+ Sneakers::CONFIG[:log].must_equal('foobar.log')
49
50
  Sneakers.clear!
50
- Sneakers::Config[:log].must_equal(STDOUT)
51
+ Sneakers::CONFIG[:log].must_equal(STDOUT)
51
52
  end
52
53
  end
53
54