sneakers_custom_bunny 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.travis.yml +6 -0
- data/CHANGELOG.md +20 -0
- data/Gemfile +3 -0
- data/Guardfile +8 -0
- data/LICENSE.txt +22 -0
- data/README.md +172 -0
- data/ROADMAP.md +18 -0
- data/Rakefile +11 -0
- data/bin/sneakers +5 -0
- data/examples/benchmark_worker.rb +20 -0
- data/examples/max_retry_handler.rb +78 -0
- data/examples/metrics_worker.rb +28 -0
- data/examples/newrelic_metrics_worker.rb +40 -0
- data/examples/profiling_worker.rb +69 -0
- data/examples/sneakers.conf.rb.example +11 -0
- data/examples/title_scraper.rb +23 -0
- data/examples/workflow_worker.rb +23 -0
- data/lib/sneakers.rb +83 -0
- data/lib/sneakers/cli.rb +115 -0
- data/lib/sneakers/concerns/logging.rb +34 -0
- data/lib/sneakers/concerns/metrics.rb +34 -0
- data/lib/sneakers/configuration.rb +59 -0
- data/lib/sneakers/handlers/maxretry.rb +191 -0
- data/lib/sneakers/handlers/oneshot.rb +30 -0
- data/lib/sneakers/metrics/logging_metrics.rb +16 -0
- data/lib/sneakers/metrics/newrelic_metrics.rb +37 -0
- data/lib/sneakers/metrics/null_metrics.rb +13 -0
- data/lib/sneakers/metrics/statsd_metrics.rb +21 -0
- data/lib/sneakers/publisher.rb +34 -0
- data/lib/sneakers/queue.rb +65 -0
- data/lib/sneakers/runner.rb +82 -0
- data/lib/sneakers/spawner.rb +27 -0
- data/lib/sneakers/support/production_formatter.rb +11 -0
- data/lib/sneakers/support/utils.rb +18 -0
- data/lib/sneakers/tasks.rb +34 -0
- data/lib/sneakers/version.rb +3 -0
- data/lib/sneakers/worker.rb +151 -0
- data/lib/sneakers/workergroup.rb +47 -0
- data/sneakers.gemspec +35 -0
- data/spec/fixtures/require_worker.rb +17 -0
- data/spec/sneakers/cli_spec.rb +63 -0
- data/spec/sneakers/concerns/logging_spec.rb +39 -0
- data/spec/sneakers/concerns/metrics_spec.rb +38 -0
- data/spec/sneakers/configuration_spec.rb +75 -0
- data/spec/sneakers/publisher_spec.rb +83 -0
- data/spec/sneakers/queue_spec.rb +115 -0
- data/spec/sneakers/runner_spec.rb +26 -0
- data/spec/sneakers/sneakers_spec.rb +75 -0
- data/spec/sneakers/support/utils_spec.rb +44 -0
- data/spec/sneakers/worker_handlers_spec.rb +390 -0
- data/spec/sneakers/worker_spec.rb +463 -0
- data/spec/spec_helper.rb +13 -0
- metadata +306 -0
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sneakers'
|
3
|
+
require 'logger'
|
4
|
+
|
5
|
+
|
6
|
+
class Foometrics
|
7
|
+
include Sneakers::Concerns::Metrics
|
8
|
+
end
|
9
|
+
|
10
|
+
describe Sneakers::Concerns::Metrics do
|
11
|
+
describe ".configure" do
|
12
|
+
before do
|
13
|
+
Foometrics.metrics = nil
|
14
|
+
end
|
15
|
+
|
16
|
+
it "should configure a default logger when included" do
|
17
|
+
Foometrics.metrics.must_be_nil
|
18
|
+
Foometrics.configure_metrics
|
19
|
+
Foometrics.metrics.wont_be_nil
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should supply accessible instance logger" do
|
23
|
+
Foometrics.metrics.must_be_nil
|
24
|
+
Foometrics.configure_metrics
|
25
|
+
f = Foometrics.new
|
26
|
+
f.metrics.must_equal Foometrics.metrics
|
27
|
+
f.metrics.wont_be_nil
|
28
|
+
end
|
29
|
+
|
30
|
+
it "should configure a given metrics when specified" do
|
31
|
+
Foometrics.metrics.must_be_nil
|
32
|
+
o = Object.new
|
33
|
+
Foometrics.configure_metrics(o)
|
34
|
+
Foometrics.metrics.must_equal o
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Sneakers::Configuration do
|
4
|
+
|
5
|
+
it 'should assign a default value for :amqp' do
|
6
|
+
with_env('RABBITMQ_URL', nil) do
|
7
|
+
config = Sneakers::Configuration.new
|
8
|
+
config[:amqp].must_equal 'amqp://guest:guest@localhost:5672'
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'should assign a default value for :vhost' do
|
13
|
+
with_env('RABBITMQ_URL', nil) do
|
14
|
+
config = Sneakers::Configuration.new
|
15
|
+
config[:vhost].must_equal '/'
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
it 'should read the value for amqp from RABBITMQ_URL' do
|
20
|
+
url = 'amqp://foo:bar@localhost:5672'
|
21
|
+
with_env('RABBITMQ_URL', url) do
|
22
|
+
config = Sneakers::Configuration.new
|
23
|
+
config[:amqp].must_equal url
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'should read the value for vhost from RABBITMQ_URL' do
|
28
|
+
url = 'amqp://foo:bar@localhost:5672/foobarvhost'
|
29
|
+
with_env('RABBITMQ_URL', url) do
|
30
|
+
config = Sneakers::Configuration.new
|
31
|
+
config[:vhost].must_equal 'foobarvhost'
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'should parse vhost from amqp option' do
|
36
|
+
env_url = 'amqp://foo:bar@localhost:5672/foobarvhost'
|
37
|
+
with_env('RABBITMQ_URL', env_url) do
|
38
|
+
url = 'amqp://foo:bar@localhost:5672/testvhost'
|
39
|
+
config = Sneakers::Configuration.new
|
40
|
+
config.merge!({ :amqp => url })
|
41
|
+
config[:vhost].must_equal 'testvhost'
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
it 'should not parse vhost from amqp option if vhost is specified explicitly' do
|
46
|
+
url = 'amqp://foo:bar@localhost:5672/foobarvhost'
|
47
|
+
config = Sneakers::Configuration.new
|
48
|
+
config.merge!({ :amqp => url, :vhost => 'test_host' })
|
49
|
+
config[:vhost].must_equal 'test_host'
|
50
|
+
end
|
51
|
+
|
52
|
+
it 'should use vhost option if it is specified' do
|
53
|
+
url = 'amqp://foo:bar@localhost:5672/foobarvhost'
|
54
|
+
with_env('RABBITMQ_URL', url) do
|
55
|
+
config = Sneakers::Configuration.new
|
56
|
+
config.merge!({ :vhost => 'test_host' })
|
57
|
+
config[:vhost].must_equal 'test_host'
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
it 'should use default vhost if vhost is not specified in amqp option' do
|
62
|
+
url = 'amqp://foo:bar@localhost:5672'
|
63
|
+
config = Sneakers::Configuration.new
|
64
|
+
config.merge!({ :amqp => url })
|
65
|
+
config[:vhost].must_equal '/'
|
66
|
+
end
|
67
|
+
|
68
|
+
def with_env(key, value)
|
69
|
+
old_value = ENV[key]
|
70
|
+
ENV[key] = value
|
71
|
+
yield
|
72
|
+
ensure
|
73
|
+
ENV[key] = old_value
|
74
|
+
end
|
75
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sneakers'
|
3
|
+
|
4
|
+
describe Sneakers::Publisher do
|
5
|
+
describe '#publish' do
|
6
|
+
before do
|
7
|
+
Sneakers.clear!
|
8
|
+
Sneakers.configure(:log => 'sneakers.log')
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should publish a message to an exchange' do
|
12
|
+
xchg = Object.new
|
13
|
+
mock(xchg).publish('test msg', routing_key: 'downloads')
|
14
|
+
|
15
|
+
p = Sneakers::Publisher.new
|
16
|
+
p.instance_variable_set(:@exchange, xchg)
|
17
|
+
|
18
|
+
mock(p).ensure_connection! {}
|
19
|
+
p.publish('test msg', to_queue: 'downloads')
|
20
|
+
end
|
21
|
+
|
22
|
+
it 'should publish with the persistence specified' do
|
23
|
+
xchg = Object.new
|
24
|
+
mock(xchg).publish('test msg', routing_key: 'downloads', persistence: true)
|
25
|
+
|
26
|
+
p = Sneakers::Publisher.new
|
27
|
+
p.instance_variable_set(:@exchange, xchg)
|
28
|
+
|
29
|
+
mock(p).ensure_connection! {}
|
30
|
+
p.publish('test msg', to_queue: 'downloads', persistence: true)
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'should publish with arbitrary metadata specified' do
|
34
|
+
xchg = Object.new
|
35
|
+
mock(xchg).publish('test msg', routing_key: 'downloads', expiration: 1, headers: {foo: 'bar'})
|
36
|
+
|
37
|
+
p = Sneakers::Publisher.new
|
38
|
+
p.instance_variable_set(:@exchange, xchg)
|
39
|
+
|
40
|
+
mock(p).ensure_connection! {}
|
41
|
+
p.publish('test msg', to_queue: 'downloads', expiration: 1, headers: {foo: 'bar'})
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'should not reconnect if already connected' do
|
45
|
+
xchg = Object.new
|
46
|
+
mock(xchg).publish('test msg', routing_key: 'downloads')
|
47
|
+
|
48
|
+
p = Sneakers::Publisher.new
|
49
|
+
p.instance_variable_set(:@exchange, xchg)
|
50
|
+
|
51
|
+
mock(p).connected? { true }
|
52
|
+
mock(p).ensure_connection!.times(0)
|
53
|
+
|
54
|
+
p.publish('test msg', to_queue: 'downloads')
|
55
|
+
end
|
56
|
+
|
57
|
+
it 'should connect to rabbitmq configured on Sneakers.configure' do
|
58
|
+
logger = Logger.new('/dev/null')
|
59
|
+
Sneakers.configure(
|
60
|
+
amqp: 'amqp://someuser:somepassword@somehost:5672',
|
61
|
+
heartbeat: 1, exchange: 'another_exchange',
|
62
|
+
exchange_type: :topic,
|
63
|
+
log: logger,
|
64
|
+
durable: false)
|
65
|
+
|
66
|
+
channel = Object.new
|
67
|
+
mock(channel).exchange('another_exchange', type: :topic, durable: false) do
|
68
|
+
mock(Object.new).publish('test msg', routing_key: 'downloads')
|
69
|
+
end
|
70
|
+
|
71
|
+
bunny = Object.new
|
72
|
+
mock(bunny).start
|
73
|
+
mock(bunny).create_channel { channel }
|
74
|
+
|
75
|
+
mock(Bunny).new('amqp://someuser:somepassword@somehost:5672', heartbeat: 1, vhost: '/', logger: logger) { bunny }
|
76
|
+
|
77
|
+
p = Sneakers::Publisher.new
|
78
|
+
|
79
|
+
p.publish('test msg', to_queue: 'downloads')
|
80
|
+
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,115 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sneakers'
|
3
|
+
|
4
|
+
|
5
|
+
|
6
|
+
describe Sneakers::Queue do
|
7
|
+
let :queue_vars do
|
8
|
+
{
|
9
|
+
:prefetch => 25,
|
10
|
+
:durable => true,
|
11
|
+
:ack => true,
|
12
|
+
:heartbeat => 2,
|
13
|
+
:vhost => '/',
|
14
|
+
:exchange => "sneakers",
|
15
|
+
:exchange_type => :direct
|
16
|
+
}
|
17
|
+
end
|
18
|
+
|
19
|
+
before do
|
20
|
+
Sneakers.configure
|
21
|
+
|
22
|
+
@mkbunny = Object.new
|
23
|
+
@mkchan = Object.new
|
24
|
+
@mkex = Object.new
|
25
|
+
@mkqueue = Object.new
|
26
|
+
@mkqueue_nondurable = Object.new
|
27
|
+
@mkworker = Object.new
|
28
|
+
|
29
|
+
mock(@mkbunny).start {}
|
30
|
+
mock(@mkbunny).create_channel{ @mkchan }
|
31
|
+
mock(Bunny).new(anything, :vhost => '/', :heartbeat => 2){ @mkbunny }
|
32
|
+
|
33
|
+
mock(@mkchan).prefetch(25)
|
34
|
+
|
35
|
+
stub(@mkworker).opts { { :exchange => 'test-exchange' } }
|
36
|
+
end
|
37
|
+
|
38
|
+
describe "#subscribe with sneakers exchange" do
|
39
|
+
before do
|
40
|
+
mock(@mkchan).exchange("sneakers", :type => :direct, :durable => true){ @mkex }
|
41
|
+
end
|
42
|
+
|
43
|
+
it "should setup a bunny queue according to configuration values" do
|
44
|
+
mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
|
45
|
+
q = Sneakers::Queue.new("downloads", queue_vars)
|
46
|
+
|
47
|
+
mock(@mkqueue).bind(@mkex, :routing_key => "downloads")
|
48
|
+
mock(@mkqueue).subscribe(:block => false, :manual_ack => true)
|
49
|
+
|
50
|
+
q.subscribe(@mkworker)
|
51
|
+
end
|
52
|
+
|
53
|
+
it "supports multiple routing_keys" do
|
54
|
+
mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
|
55
|
+
q = Sneakers::Queue.new("downloads",
|
56
|
+
queue_vars.merge(:routing_key => ["alpha", "beta"]))
|
57
|
+
|
58
|
+
mock(@mkqueue).bind(@mkex, :routing_key => "alpha")
|
59
|
+
mock(@mkqueue).bind(@mkex, :routing_key => "beta")
|
60
|
+
mock(@mkqueue).subscribe(:block => false, :manual_ack => true)
|
61
|
+
|
62
|
+
q.subscribe(@mkworker)
|
63
|
+
end
|
64
|
+
|
65
|
+
it "will use whatever handler the worker specifies" do
|
66
|
+
mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
|
67
|
+
@handler = Object.new
|
68
|
+
worker_opts = { :handler => @handler }
|
69
|
+
stub(@mkworker).opts { worker_opts }
|
70
|
+
mock(@handler).new(@mkchan, @mkqueue, worker_opts).once
|
71
|
+
|
72
|
+
stub(@mkqueue).bind
|
73
|
+
stub(@mkqueue).subscribe
|
74
|
+
q = Sneakers::Queue.new("downloads", queue_vars)
|
75
|
+
q.subscribe(@mkworker)
|
76
|
+
end
|
77
|
+
|
78
|
+
it "creates a non-durable queue if :queue_durable => false" do
|
79
|
+
mock(@mkchan).queue("test_nondurable", :durable => false) { @mkqueue_nondurable }
|
80
|
+
queue_vars[:queue_durable] = false
|
81
|
+
q = Sneakers::Queue.new("test_nondurable", queue_vars)
|
82
|
+
|
83
|
+
mock(@mkqueue_nondurable).bind(@mkex, :routing_key => "test_nondurable")
|
84
|
+
mock(@mkqueue_nondurable).subscribe(:block => false, :manual_ack => true)
|
85
|
+
|
86
|
+
q.subscribe(@mkworker)
|
87
|
+
myqueue = q.instance_variable_get(:@queue)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
describe "#subscribe with default exchange" do
|
92
|
+
before do
|
93
|
+
# expect default exchange
|
94
|
+
queue_vars[:exchange] = ""
|
95
|
+
mock(@mkchan).exchange("", :type => :direct, :durable => true){ @mkex }
|
96
|
+
end
|
97
|
+
|
98
|
+
it "does not bind to exchange" do
|
99
|
+
mock(@mkchan).queue("downloads", :durable => true) { @mkqueue }
|
100
|
+
@handler = Object.new
|
101
|
+
worker_opts = { :handler => @handler }
|
102
|
+
stub(@mkworker).opts { worker_opts }
|
103
|
+
mock(@handler).new(@mkchan, @mkqueue, worker_opts).once
|
104
|
+
|
105
|
+
stub(@mkqueue).bind do
|
106
|
+
raise "bind should not be called"
|
107
|
+
end
|
108
|
+
|
109
|
+
stub(@mkqueue).subscribe
|
110
|
+
q = Sneakers::Queue.new("downloads", queue_vars)
|
111
|
+
q.subscribe(@mkworker)
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'spec_helper'
|
3
|
+
require 'sneakers'
|
4
|
+
|
5
|
+
describe Sneakers::Runner do
|
6
|
+
let(:logger) { Logger.new('logtest.log') }
|
7
|
+
|
8
|
+
describe "with configuration that specifies a logger object" do
|
9
|
+
before do
|
10
|
+
Sneakers.configure(log: logger)
|
11
|
+
@runner = Sneakers::Runner.new([])
|
12
|
+
end
|
13
|
+
|
14
|
+
it 'passes the logger to serverengine' do
|
15
|
+
# Stub out ServerEngine::Daemon.run so we only exercise the way we invoke
|
16
|
+
# ServerEngine.create
|
17
|
+
any_instance_of(ServerEngine::Daemon) do |daemon|
|
18
|
+
stub(daemon).main{ return 0 }
|
19
|
+
end
|
20
|
+
|
21
|
+
@runner.run
|
22
|
+
# look at @runner's @se instance variable (actually of type Daemon)...and
|
23
|
+
# figure out what it's logger is...
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sneakers'
|
3
|
+
|
4
|
+
class EnvWorker
|
5
|
+
include Sneakers::Worker
|
6
|
+
from_queue 'defaults'
|
7
|
+
|
8
|
+
def work(msg)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
|
13
|
+
describe Sneakers do
|
14
|
+
before do
|
15
|
+
Sneakers.clear!
|
16
|
+
end
|
17
|
+
|
18
|
+
describe 'self' do
|
19
|
+
it 'should have defaults set up' do
|
20
|
+
Sneakers::CONFIG[:log].must_equal(STDOUT)
|
21
|
+
end
|
22
|
+
|
23
|
+
it 'should configure itself' do
|
24
|
+
Sneakers.configure
|
25
|
+
Sneakers.logger.wont_be_nil
|
26
|
+
Sneakers.configured?.must_equal(true)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
describe '.daemonize!' do
|
31
|
+
it 'should set a logger to a default info level and not daemonize' do
|
32
|
+
Sneakers.daemonize!
|
33
|
+
Sneakers::CONFIG[:log].must_equal('sneakers.log')
|
34
|
+
Sneakers::CONFIG[:daemonize].must_equal(true)
|
35
|
+
Sneakers.logger.level.must_equal(Logger::INFO)
|
36
|
+
end
|
37
|
+
|
38
|
+
it 'should set a logger to a level given that level' do
|
39
|
+
Sneakers.daemonize!(Logger::DEBUG)
|
40
|
+
Sneakers.logger.level.must_equal(Logger::DEBUG)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
|
45
|
+
describe '.clear!' do
|
46
|
+
it 'must reset dirty configuration to default' do
|
47
|
+
Sneakers::CONFIG[:log].must_equal(STDOUT)
|
48
|
+
Sneakers.configure(:log => 'foobar.log')
|
49
|
+
Sneakers::CONFIG[:log].must_equal('foobar.log')
|
50
|
+
Sneakers.clear!
|
51
|
+
Sneakers::CONFIG[:log].must_equal(STDOUT)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
|
56
|
+
describe '#setup_general_logger' do
|
57
|
+
it 'should detect a string and configure a logger' do
|
58
|
+
Sneakers.configure(:log => 'sneakers.log')
|
59
|
+
Sneakers.logger.kind_of?(Logger).must_equal(true)
|
60
|
+
end
|
61
|
+
|
62
|
+
it 'should detect a file-like thing and configure a logger' do
|
63
|
+
Sneakers.configure(:log => STDOUT)
|
64
|
+
Sneakers.logger.kind_of?(Logger).must_equal(true)
|
65
|
+
end
|
66
|
+
|
67
|
+
it 'should detect an actual logger and configure it' do
|
68
|
+
logger = Logger.new(STDOUT)
|
69
|
+
Sneakers.configure(:log => logger)
|
70
|
+
Sneakers.logger.must_equal(logger)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sneakers'
|
3
|
+
|
4
|
+
describe Sneakers::Utils do
|
5
|
+
describe '::parse_workers' do
|
6
|
+
before(:all) do
|
7
|
+
class Foo; end
|
8
|
+
class Bar; end
|
9
|
+
class Baz
|
10
|
+
class Quux; end
|
11
|
+
class Corge; end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
describe 'given a single class name' do
|
16
|
+
describe 'without namespace' do
|
17
|
+
it 'returns the worker class name' do
|
18
|
+
Sneakers::Utils.parse_workers('Foo').must_equal([[Foo],[]])
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe 'with namespace' do
|
23
|
+
it 'returns the worker class name' do
|
24
|
+
Sneakers::Utils.parse_workers('Baz::Quux').must_equal([[Baz::Quux],[]])
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
describe 'given a list of class names' do
|
30
|
+
describe 'without namespaces' do
|
31
|
+
it 'returns all worker class names' do
|
32
|
+
Sneakers::Utils.parse_workers('Foo,Bar').must_equal([[Foo,Bar],[]])
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
describe 'with namespaces' do
|
37
|
+
it 'returns all worker class names' do
|
38
|
+
workers = Sneakers::Utils.parse_workers('Baz::Quux,Baz::Corge')
|
39
|
+
workers.must_equal([[Baz::Quux,Baz::Corge],[]])
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|