sneakers_custom_bunny 1.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.travis.yml +6 -0
- data/CHANGELOG.md +20 -0
- data/Gemfile +3 -0
- data/Guardfile +8 -0
- data/LICENSE.txt +22 -0
- data/README.md +172 -0
- data/ROADMAP.md +18 -0
- data/Rakefile +11 -0
- data/bin/sneakers +5 -0
- data/examples/benchmark_worker.rb +20 -0
- data/examples/max_retry_handler.rb +78 -0
- data/examples/metrics_worker.rb +28 -0
- data/examples/newrelic_metrics_worker.rb +40 -0
- data/examples/profiling_worker.rb +69 -0
- data/examples/sneakers.conf.rb.example +11 -0
- data/examples/title_scraper.rb +23 -0
- data/examples/workflow_worker.rb +23 -0
- data/lib/sneakers.rb +83 -0
- data/lib/sneakers/cli.rb +115 -0
- data/lib/sneakers/concerns/logging.rb +34 -0
- data/lib/sneakers/concerns/metrics.rb +34 -0
- data/lib/sneakers/configuration.rb +59 -0
- data/lib/sneakers/handlers/maxretry.rb +191 -0
- data/lib/sneakers/handlers/oneshot.rb +30 -0
- data/lib/sneakers/metrics/logging_metrics.rb +16 -0
- data/lib/sneakers/metrics/newrelic_metrics.rb +37 -0
- data/lib/sneakers/metrics/null_metrics.rb +13 -0
- data/lib/sneakers/metrics/statsd_metrics.rb +21 -0
- data/lib/sneakers/publisher.rb +34 -0
- data/lib/sneakers/queue.rb +65 -0
- data/lib/sneakers/runner.rb +82 -0
- data/lib/sneakers/spawner.rb +27 -0
- data/lib/sneakers/support/production_formatter.rb +11 -0
- data/lib/sneakers/support/utils.rb +18 -0
- data/lib/sneakers/tasks.rb +34 -0
- data/lib/sneakers/version.rb +3 -0
- data/lib/sneakers/worker.rb +151 -0
- data/lib/sneakers/workergroup.rb +47 -0
- data/sneakers.gemspec +35 -0
- data/spec/fixtures/require_worker.rb +17 -0
- data/spec/sneakers/cli_spec.rb +63 -0
- data/spec/sneakers/concerns/logging_spec.rb +39 -0
- data/spec/sneakers/concerns/metrics_spec.rb +38 -0
- data/spec/sneakers/configuration_spec.rb +75 -0
- data/spec/sneakers/publisher_spec.rb +83 -0
- data/spec/sneakers/queue_spec.rb +115 -0
- data/spec/sneakers/runner_spec.rb +26 -0
- data/spec/sneakers/sneakers_spec.rb +75 -0
- data/spec/sneakers/support/utils_spec.rb +44 -0
- data/spec/sneakers/worker_handlers_spec.rb +390 -0
- data/spec/sneakers/worker_spec.rb +463 -0
- data/spec/spec_helper.rb +13 -0
- metadata +306 -0
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sneakers'
|
3
|
+
require 'logger'
|
4
|
+
|
5
|
+
|
6
|
+
class Foometrics
|
7
|
+
include Sneakers::Concerns::Metrics
|
8
|
+
end
|
9
|
+
|
10
|
+
describe Sneakers::Concerns::Metrics do
|
11
|
+
describe ".configure" do
|
12
|
+
before do
|
13
|
+
Foometrics.metrics = nil
|
14
|
+
end
|
15
|
+
|
16
|
+
it "should configure a default logger when included" do
|
17
|
+
Foometrics.metrics.must_be_nil
|
18
|
+
Foometrics.configure_metrics
|
19
|
+
Foometrics.metrics.wont_be_nil
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should supply accessible instance logger" do
|
23
|
+
Foometrics.metrics.must_be_nil
|
24
|
+
Foometrics.configure_metrics
|
25
|
+
f = Foometrics.new
|
26
|
+
f.metrics.must_equal Foometrics.metrics
|
27
|
+
f.metrics.wont_be_nil
|
28
|
+
end
|
29
|
+
|
30
|
+
it "should configure a given metrics when specified" do
|
31
|
+
Foometrics.metrics.must_be_nil
|
32
|
+
o = Object.new
|
33
|
+
Foometrics.configure_metrics(o)
|
34
|
+
Foometrics.metrics.must_equal o
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Sneakers::Configuration do
|
4
|
+
|
5
|
+
it 'should assign a default value for :amqp' do
|
6
|
+
with_env('RABBITMQ_URL', nil) do
|
7
|
+
config = Sneakers::Configuration.new
|
8
|
+
config[:amqp].must_equal 'amqp://guest:guest@localhost:5672'
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'should assign a default value for :vhost' do
|
13
|
+
with_env('RABBITMQ_URL', nil) do
|
14
|
+
config = Sneakers::Configuration.new
|
15
|
+
config[:vhost].must_equal '/'
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
it 'should read the value for amqp from RABBITMQ_URL' do
|
20
|
+
url = 'amqp://foo:bar@localhost:5672'
|
21
|
+
with_env('RABBITMQ_URL', url) do
|
22
|
+
config = Sneakers::Configuration.new
|
23
|
+
config[:amqp].must_equal url
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'should read the value for vhost from RABBITMQ_URL' do
|
28
|
+
url = 'amqp://foo:bar@localhost:5672/foobarvhost'
|
29
|
+
with_env('RABBITMQ_URL', url) do
|
30
|
+
config = Sneakers::Configuration.new
|
31
|
+
config[:vhost].must_equal 'foobarvhost'
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'should parse vhost from amqp option' do
|
36
|
+
env_url = 'amqp://foo:bar@localhost:5672/foobarvhost'
|
37
|
+
with_env('RABBITMQ_URL', env_url) do
|
38
|
+
url = 'amqp://foo:bar@localhost:5672/testvhost'
|
39
|
+
config = Sneakers::Configuration.new
|
40
|
+
config.merge!({ :amqp => url })
|
41
|
+
config[:vhost].must_equal 'testvhost'
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
it 'should not parse vhost from amqp option if vhost is specified explicitly' do
|
46
|
+
url = 'amqp://foo:bar@localhost:5672/foobarvhost'
|
47
|
+
config = Sneakers::Configuration.new
|
48
|
+
config.merge!({ :amqp => url, :vhost => 'test_host' })
|
49
|
+
config[:vhost].must_equal 'test_host'
|
50
|
+
end
|
51
|
+
|
52
|
+
it 'should use vhost option if it is specified' do
|
53
|
+
url = 'amqp://foo:bar@localhost:5672/foobarvhost'
|
54
|
+
with_env('RABBITMQ_URL', url) do
|
55
|
+
config = Sneakers::Configuration.new
|
56
|
+
config.merge!({ :vhost => 'test_host' })
|
57
|
+
config[:vhost].must_equal 'test_host'
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
it 'should use default vhost if vhost is not specified in amqp option' do
|
62
|
+
url = 'amqp://foo:bar@localhost:5672'
|
63
|
+
config = Sneakers::Configuration.new
|
64
|
+
config.merge!({ :amqp => url })
|
65
|
+
config[:vhost].must_equal '/'
|
66
|
+
end
|
67
|
+
|
68
|
+
def with_env(key, value)
|
69
|
+
old_value = ENV[key]
|
70
|
+
ENV[key] = value
|
71
|
+
yield
|
72
|
+
ensure
|
73
|
+
ENV[key] = old_value
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sneakers'
|
3
|
+
|
4
|
+
describe Sneakers::Publisher do
|
5
|
+
describe '#publish' do
|
6
|
+
before do
|
7
|
+
Sneakers.clear!
|
8
|
+
Sneakers.configure(:log => 'sneakers.log')
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should publish a message to an exchange' do
|
12
|
+
xchg = Object.new
|
13
|
+
mock(xchg).publish('test msg', routing_key: 'downloads')
|
14
|
+
|
15
|
+
p = Sneakers::Publisher.new
|
16
|
+
p.instance_variable_set(:@exchange, xchg)
|
17
|
+
|
18
|
+
mock(p).ensure_connection! {}
|
19
|
+
p.publish('test msg', to_queue: 'downloads')
|
20
|
+
end
|
21
|
+
|
22
|
+
it 'should publish with the persistence specified' do
|
23
|
+
xchg = Object.new
|
24
|
+
mock(xchg).publish('test msg', routing_key: 'downloads', persistence: true)
|
25
|
+
|
26
|
+
p = Sneakers::Publisher.new
|
27
|
+
p.instance_variable_set(:@exchange, xchg)
|
28
|
+
|
29
|
+
mock(p).ensure_connection! {}
|
30
|
+
p.publish('test msg', to_queue: 'downloads', persistence: true)
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'should publish with arbitrary metadata specified' do
|
34
|
+
xchg = Object.new
|
35
|
+
mock(xchg).publish('test msg', routing_key: 'downloads', expiration: 1, headers: {foo: 'bar'})
|
36
|
+
|
37
|
+
p = Sneakers::Publisher.new
|
38
|
+
p.instance_variable_set(:@exchange, xchg)
|
39
|
+
|
40
|
+
mock(p).ensure_connection! {}
|
41
|
+
p.publish('test msg', to_queue: 'downloads', expiration: 1, headers: {foo: 'bar'})
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'should not reconnect if already connected' do
|
45
|
+
xchg = Object.new
|
46
|
+
mock(xchg).publish('test msg', routing_key: 'downloads')
|
47
|
+
|
48
|
+
p = Sneakers::Publisher.new
|
49
|
+
p.instance_variable_set(:@exchange, xchg)
|
50
|
+
|
51
|
+
mock(p).connected? { true }
|
52
|
+
mock(p).ensure_connection!.times(0)
|
53
|
+
|
54
|
+
p.publish('test msg', to_queue: 'downloads')
|
55
|
+
end
|
56
|
+
|
57
|
+
it 'should connect to rabbitmq configured on Sneakers.configure' do
|
58
|
+
logger = Logger.new('/dev/null')
|
59
|
+
Sneakers.configure(
|
60
|
+
amqp: 'amqp://someuser:somepassword@somehost:5672',
|
61
|
+
heartbeat: 1, exchange: 'another_exchange',
|
62
|
+
exchange_type: :topic,
|
63
|
+
log: logger,
|
64
|
+
durable: false)
|
65
|
+
|
66
|
+
channel = Object.new
|
67
|
+
mock(channel).exchange('another_exchange', type: :topic, durable: false) do
|
68
|
+
mock(Object.new).publish('test msg', routing_key: 'downloads')
|
69
|
+
end
|
70
|
+
|
71
|
+
bunny = Object.new
|
72
|
+
mock(bunny).start
|
73
|
+
mock(bunny).create_channel { channel }
|
74
|
+
|
75
|
+
mock(Bunny).new('amqp://someuser:somepassword@somehost:5672', heartbeat: 1, vhost: '/', logger: logger) { bunny }
|
76
|
+
|
77
|
+
p = Sneakers::Publisher.new
|
78
|
+
|
79
|
+
p.publish('test msg', to_queue: 'downloads')
|
80
|
+
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,115 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sneakers'
|
3
|
+
|
4
|
+
|
5
|
+
|
6
|
+
describe Sneakers::Queue do
|
7
|
+
let :queue_vars do
|
8
|
+
{
|
9
|
+
:prefetch => 25,
|
10
|
+
:durable => true,
|
11
|
+
:ack => true,
|
12
|
+
:heartbeat => 2,
|
13
|
+
:vhost => '/',
|
14
|
+
:exchange => "sneakers",
|
15
|
+
:exchange_type => :direct
|
16
|
+
}
|
17
|
+
end
|
18
|
+
|
19
|
+
before do
|
20
|
+
Sneakers.configure
|
21
|
+
|
22
|
+
@mkbunny = Object.new
|
23
|
+
@mkchan = Object.new
|
24
|
+
@mkex = Object.new
|
25
|
+
@mkqueue = Object.new
|
26
|
+
@mkqueue_nondurable = Object.new
|
27
|
+
@mkworker = Object.new
|
28
|
+
|
29
|
+
mock(@mkbunny).start {}
|
30
|
+
mock(@mkbunny).create_channel{ @mkchan }
|
31
|
+
mock(Bunny).new(anything, :vhost => '/', :heartbeat => 2){ @mkbunny }
|
32
|
+
|
33
|
+
mock(@mkchan).prefetch(25)
|
34
|
+
|
35
|
+
stub(@mkworker).opts { { :exchange => 'test-exchange' } }
|
36
|
+
end
|
37
|
+
|
38
|
+
describe "#subscribe with sneakers exchange" do
|
39
|
+
before do
|
40
|
+
mock(@mkchan).exchange("sneakers", :type => :direct, :durable => true){ @mkex }
|
41
|
+
end
|
42
|
+
|
43
|
+
it "should setup a bunny queue according to configuration values" do
|
44
|
+
mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
|
45
|
+
q = Sneakers::Queue.new("downloads", queue_vars)
|
46
|
+
|
47
|
+
mock(@mkqueue).bind(@mkex, :routing_key => "downloads")
|
48
|
+
mock(@mkqueue).subscribe(:block => false, :manual_ack => true)
|
49
|
+
|
50
|
+
q.subscribe(@mkworker)
|
51
|
+
end
|
52
|
+
|
53
|
+
it "supports multiple routing_keys" do
|
54
|
+
mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
|
55
|
+
q = Sneakers::Queue.new("downloads",
|
56
|
+
queue_vars.merge(:routing_key => ["alpha", "beta"]))
|
57
|
+
|
58
|
+
mock(@mkqueue).bind(@mkex, :routing_key => "alpha")
|
59
|
+
mock(@mkqueue).bind(@mkex, :routing_key => "beta")
|
60
|
+
mock(@mkqueue).subscribe(:block => false, :manual_ack => true)
|
61
|
+
|
62
|
+
q.subscribe(@mkworker)
|
63
|
+
end
|
64
|
+
|
65
|
+
it "will use whatever handler the worker specifies" do
|
66
|
+
mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
|
67
|
+
@handler = Object.new
|
68
|
+
worker_opts = { :handler => @handler }
|
69
|
+
stub(@mkworker).opts { worker_opts }
|
70
|
+
mock(@handler).new(@mkchan, @mkqueue, worker_opts).once
|
71
|
+
|
72
|
+
stub(@mkqueue).bind
|
73
|
+
stub(@mkqueue).subscribe
|
74
|
+
q = Sneakers::Queue.new("downloads", queue_vars)
|
75
|
+
q.subscribe(@mkworker)
|
76
|
+
end
|
77
|
+
|
78
|
+
it "creates a non-durable queue if :queue_durable => false" do
|
79
|
+
mock(@mkchan).queue("test_nondurable", :durable => false) { @mkqueue_nondurable }
|
80
|
+
queue_vars[:queue_durable] = false
|
81
|
+
q = Sneakers::Queue.new("test_nondurable", queue_vars)
|
82
|
+
|
83
|
+
mock(@mkqueue_nondurable).bind(@mkex, :routing_key => "test_nondurable")
|
84
|
+
mock(@mkqueue_nondurable).subscribe(:block => false, :manual_ack => true)
|
85
|
+
|
86
|
+
q.subscribe(@mkworker)
|
87
|
+
myqueue = q.instance_variable_get(:@queue)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
describe "#subscribe with default exchange" do
|
92
|
+
before do
|
93
|
+
# expect default exchange
|
94
|
+
queue_vars[:exchange] = ""
|
95
|
+
mock(@mkchan).exchange("", :type => :direct, :durable => true){ @mkex }
|
96
|
+
end
|
97
|
+
|
98
|
+
it "does not bind to exchange" do
|
99
|
+
mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
|
100
|
+
@handler = Object.new
|
101
|
+
worker_opts = { :handler => @handler }
|
102
|
+
stub(@mkworker).opts { worker_opts }
|
103
|
+
mock(@handler).new(@mkchan, @mkqueue, worker_opts).once
|
104
|
+
|
105
|
+
stub(@mkqueue).bind do
|
106
|
+
raise "bind should not be called"
|
107
|
+
end
|
108
|
+
|
109
|
+
stub(@mkqueue).subscribe
|
110
|
+
q = Sneakers::Queue.new("downloads", queue_vars)
|
111
|
+
q.subscribe(@mkworker)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'spec_helper'
|
3
|
+
require 'sneakers'
|
4
|
+
|
5
|
+
describe Sneakers::Runner do
|
6
|
+
let(:logger) { Logger.new('logtest.log') }
|
7
|
+
|
8
|
+
describe "with configuration that specifies a logger object" do
|
9
|
+
before do
|
10
|
+
Sneakers.configure(log: logger)
|
11
|
+
@runner = Sneakers::Runner.new([])
|
12
|
+
end
|
13
|
+
|
14
|
+
it 'passes the logger to serverengine' do
|
15
|
+
# Stub out ServerEngine::Daemon.run so we only exercise the way we invoke
|
16
|
+
# ServerEngine.create
|
17
|
+
any_instance_of(ServerEngine::Daemon) do |daemon|
|
18
|
+
stub(daemon).main{ return 0 }
|
19
|
+
end
|
20
|
+
|
21
|
+
@runner.run
|
22
|
+
# look at @runner's @se instance variable (actually of type Daemon)...and
|
23
|
+
# figure out what it's logger is...
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sneakers'
|
3
|
+
|
4
|
+
class EnvWorker
|
5
|
+
include Sneakers::Worker
|
6
|
+
from_queue 'defaults'
|
7
|
+
|
8
|
+
def work(msg)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
|
13
|
+
describe Sneakers do
|
14
|
+
before do
|
15
|
+
Sneakers.clear!
|
16
|
+
end
|
17
|
+
|
18
|
+
describe 'self' do
|
19
|
+
it 'should have defaults set up' do
|
20
|
+
Sneakers::CONFIG[:log].must_equal(STDOUT)
|
21
|
+
end
|
22
|
+
|
23
|
+
it 'should configure itself' do
|
24
|
+
Sneakers.configure
|
25
|
+
Sneakers.logger.wont_be_nil
|
26
|
+
Sneakers.configured?.must_equal(true)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
describe '.daemonize!' do
|
31
|
+
it 'should set a logger to a default info level and not daemonize' do
|
32
|
+
Sneakers.daemonize!
|
33
|
+
Sneakers::CONFIG[:log].must_equal('sneakers.log')
|
34
|
+
Sneakers::CONFIG[:daemonize].must_equal(true)
|
35
|
+
Sneakers.logger.level.must_equal(Logger::INFO)
|
36
|
+
end
|
37
|
+
|
38
|
+
it 'should set a logger to a level given that level' do
|
39
|
+
Sneakers.daemonize!(Logger::DEBUG)
|
40
|
+
Sneakers.logger.level.must_equal(Logger::DEBUG)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
|
45
|
+
describe '.clear!' do
|
46
|
+
it 'must reset dirty configuration to default' do
|
47
|
+
Sneakers::CONFIG[:log].must_equal(STDOUT)
|
48
|
+
Sneakers.configure(:log => 'foobar.log')
|
49
|
+
Sneakers::CONFIG[:log].must_equal('foobar.log')
|
50
|
+
Sneakers.clear!
|
51
|
+
Sneakers::CONFIG[:log].must_equal(STDOUT)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
|
56
|
+
describe '#setup_general_logger' do
|
57
|
+
it 'should detect a string and configure a logger' do
|
58
|
+
Sneakers.configure(:log => 'sneakers.log')
|
59
|
+
Sneakers.logger.kind_of?(Logger).must_equal(true)
|
60
|
+
end
|
61
|
+
|
62
|
+
it 'should detect a file-like thing and configure a logger' do
|
63
|
+
Sneakers.configure(:log => STDOUT)
|
64
|
+
Sneakers.logger.kind_of?(Logger).must_equal(true)
|
65
|
+
end
|
66
|
+
|
67
|
+
it 'should detect an actual logger and configure it' do
|
68
|
+
logger = Logger.new(STDOUT)
|
69
|
+
Sneakers.configure(:log => logger)
|
70
|
+
Sneakers.logger.must_equal(logger)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sneakers'
|
3
|
+
|
4
|
+
describe Sneakers::Utils do
|
5
|
+
describe '::parse_workers' do
|
6
|
+
before(:all) do
|
7
|
+
class Foo; end
|
8
|
+
class Bar; end
|
9
|
+
class Baz
|
10
|
+
class Quux; end
|
11
|
+
class Corge; end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
describe 'given a single class name' do
|
16
|
+
describe 'without namespace' do
|
17
|
+
it 'returns the worker class name' do
|
18
|
+
Sneakers::Utils.parse_workers('Foo').must_equal([[Foo],[]])
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe 'with namespace' do
|
23
|
+
it 'returns the worker class name' do
|
24
|
+
Sneakers::Utils.parse_workers('Baz::Quux').must_equal([[Baz::Quux],[]])
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
describe 'given a list of class names' do
|
30
|
+
describe 'without namespaces' do
|
31
|
+
it 'returns all worker class names' do
|
32
|
+
Sneakers::Utils.parse_workers('Foo,Bar').must_equal([[Foo,Bar],[]])
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
describe 'with namespaces' do
|
37
|
+
it 'returns all worker class names' do
|
38
|
+
workers = Sneakers::Utils.parse_workers('Baz::Quux,Baz::Corge')
|
39
|
+
workers.must_equal([[Baz::Quux,Baz::Corge],[]])
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|