kicks 3.0.0.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.github/workflows/ci.yml +24 -0
- data/.gitignore +12 -0
- data/ChangeLog.md +142 -0
- data/Dockerfile +24 -0
- data/Dockerfile.slim +20 -0
- data/Gemfile +8 -0
- data/Guardfile +8 -0
- data/LICENSE.txt +22 -0
- data/README.md +209 -0
- data/Rakefile +12 -0
- data/bin/sneakers +6 -0
- data/docker-compose.yml +24 -0
- data/examples/benchmark_worker.rb +22 -0
- data/examples/max_retry_handler.rb +68 -0
- data/examples/metrics_worker.rb +34 -0
- data/examples/middleware_worker.rb +36 -0
- data/examples/newrelic_metrics_worker.rb +40 -0
- data/examples/profiling_worker.rb +69 -0
- data/examples/sneakers.conf.rb.example +11 -0
- data/examples/title_scraper.rb +36 -0
- data/examples/workflow_worker.rb +23 -0
- data/kicks.gemspec +44 -0
- data/lib/sneakers/cli.rb +122 -0
- data/lib/sneakers/concerns/logging.rb +34 -0
- data/lib/sneakers/concerns/metrics.rb +34 -0
- data/lib/sneakers/configuration.rb +125 -0
- data/lib/sneakers/content_encoding.rb +47 -0
- data/lib/sneakers/content_type.rb +47 -0
- data/lib/sneakers/error_reporter.rb +33 -0
- data/lib/sneakers/errors.rb +2 -0
- data/lib/sneakers/handlers/maxretry.rb +219 -0
- data/lib/sneakers/handlers/oneshot.rb +26 -0
- data/lib/sneakers/metrics/logging_metrics.rb +16 -0
- data/lib/sneakers/metrics/newrelic_metrics.rb +32 -0
- data/lib/sneakers/metrics/null_metrics.rb +13 -0
- data/lib/sneakers/metrics/statsd_metrics.rb +21 -0
- data/lib/sneakers/middleware/config.rb +23 -0
- data/lib/sneakers/publisher.rb +49 -0
- data/lib/sneakers/queue.rb +87 -0
- data/lib/sneakers/runner.rb +91 -0
- data/lib/sneakers/spawner.rb +30 -0
- data/lib/sneakers/support/production_formatter.rb +11 -0
- data/lib/sneakers/support/utils.rb +18 -0
- data/lib/sneakers/tasks.rb +66 -0
- data/lib/sneakers/version.rb +3 -0
- data/lib/sneakers/worker.rb +162 -0
- data/lib/sneakers/workergroup.rb +60 -0
- data/lib/sneakers.rb +125 -0
- data/log/.gitkeep +0 -0
- data/scripts/local_integration +2 -0
- data/scripts/local_worker +3 -0
- data/spec/fixtures/integration_worker.rb +18 -0
- data/spec/fixtures/require_worker.rb +23 -0
- data/spec/gzip_helper.rb +15 -0
- data/spec/sneakers/cli_spec.rb +75 -0
- data/spec/sneakers/concerns/logging_spec.rb +39 -0
- data/spec/sneakers/concerns/metrics_spec.rb +38 -0
- data/spec/sneakers/configuration_spec.rb +97 -0
- data/spec/sneakers/content_encoding_spec.rb +81 -0
- data/spec/sneakers/content_type_spec.rb +81 -0
- data/spec/sneakers/integration_spec.rb +158 -0
- data/spec/sneakers/publisher_spec.rb +179 -0
- data/spec/sneakers/queue_spec.rb +169 -0
- data/spec/sneakers/runner_spec.rb +70 -0
- data/spec/sneakers/sneakers_spec.rb +77 -0
- data/spec/sneakers/support/utils_spec.rb +44 -0
- data/spec/sneakers/tasks/sneakers_run_spec.rb +115 -0
- data/spec/sneakers/worker_handlers_spec.rb +469 -0
- data/spec/sneakers/worker_spec.rb +712 -0
- data/spec/sneakers/workergroup_spec.rb +83 -0
- data/spec/spec_helper.rb +21 -0
- metadata +352 -0
@@ -0,0 +1,97 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Sneakers::Configuration do
|
4
|
+
|
5
|
+
describe 'with a connection' do
|
6
|
+
let(:connection) { Object.new }
|
7
|
+
|
8
|
+
it 'does not use vhost option if it is specified' do
|
9
|
+
url = 'amqp://foo:bar@localhost:5672/foobarvhost'
|
10
|
+
with_env('RABBITMQ_URL', url) do
|
11
|
+
config = Sneakers::Configuration.new
|
12
|
+
config.merge!({ :vhost => 'test_host', :connection => connection })
|
13
|
+
_(config.has_key?(:vhost)).must_equal false
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'does not amqp option if it is specified' do
|
18
|
+
url = 'amqp://foo:bar@localhost:5672'
|
19
|
+
config = Sneakers::Configuration.new
|
20
|
+
config.merge!({ :amqp => url, :connection => connection })
|
21
|
+
_(config.has_key?(:vhost)).must_equal false
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
describe 'without a connection' do
|
26
|
+
it 'should assign a default value for :amqp' do
|
27
|
+
with_env('RABBITMQ_URL', nil) do
|
28
|
+
config = Sneakers::Configuration.new
|
29
|
+
_(config[:amqp]).must_equal 'amqp://guest:guest@localhost:5672'
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'should assign a default value for :vhost' do
|
34
|
+
with_env('RABBITMQ_URL', nil) do
|
35
|
+
config = Sneakers::Configuration.new
|
36
|
+
_(config[:vhost]).must_equal '/'
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'should read the value for amqp from RABBITMQ_URL' do
|
41
|
+
url = 'amqp://foo:bar@localhost:5672'
|
42
|
+
with_env('RABBITMQ_URL', url) do
|
43
|
+
config = Sneakers::Configuration.new
|
44
|
+
_(config[:amqp]).must_equal url
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'should read the value for vhost from RABBITMQ_URL' do
|
49
|
+
url = 'amqp://foo:bar@localhost:5672/foobarvhost'
|
50
|
+
with_env('RABBITMQ_URL', url) do
|
51
|
+
config = Sneakers::Configuration.new
|
52
|
+
_(config[:vhost]).must_equal 'foobarvhost'
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
it 'should parse vhost from amqp option' do
|
57
|
+
env_url = 'amqp://foo:bar@localhost:5672/foobarvhost'
|
58
|
+
with_env('RABBITMQ_URL', env_url) do
|
59
|
+
url = 'amqp://foo:bar@localhost:5672/testvhost'
|
60
|
+
config = Sneakers::Configuration.new
|
61
|
+
config.merge!({ :amqp => url })
|
62
|
+
_(config[:vhost]).must_equal 'testvhost'
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
it 'should not parse vhost from amqp option if vhost is specified explicitly' do
|
67
|
+
url = 'amqp://foo:bar@localhost:5672/foobarvhost'
|
68
|
+
config = Sneakers::Configuration.new
|
69
|
+
config.merge!({ :amqp => url, :vhost => 'test_host' })
|
70
|
+
_(config[:vhost]).must_equal 'test_host'
|
71
|
+
end
|
72
|
+
|
73
|
+
it 'should use vhost option if it is specified' do
|
74
|
+
url = 'amqp://foo:bar@localhost:5672/foobarvhost'
|
75
|
+
with_env('RABBITMQ_URL', url) do
|
76
|
+
config = Sneakers::Configuration.new
|
77
|
+
config.merge!({ :vhost => 'test_host' })
|
78
|
+
_(config[:vhost]).must_equal 'test_host'
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
it 'should use default vhost if vhost is not specified in amqp option' do
|
83
|
+
url = 'amqp://foo:bar@localhost:5672'
|
84
|
+
config = Sneakers::Configuration.new
|
85
|
+
config.merge!({ :amqp => url })
|
86
|
+
_(config[:vhost]).must_equal '/'
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def with_env(key, value)
|
91
|
+
old_value = ENV[key]
|
92
|
+
ENV[key] = value
|
93
|
+
yield
|
94
|
+
ensure
|
95
|
+
ENV[key] = old_value
|
96
|
+
end
|
97
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'gzip_helper'
|
3
|
+
require 'sneakers/content_encoding'
|
4
|
+
|
5
|
+
describe Sneakers::ContentEncoding do
|
6
|
+
after do
|
7
|
+
Sneakers::ContentEncoding.reset!
|
8
|
+
end
|
9
|
+
|
10
|
+
describe '.decode' do
|
11
|
+
it 'uses the given decoder' do
|
12
|
+
Sneakers::ContentEncoding.register(
|
13
|
+
content_encoding: 'gzip',
|
14
|
+
encoder: ->(_) {},
|
15
|
+
decoder: ->(payload) { gzip_decompress(payload) },
|
16
|
+
)
|
17
|
+
|
18
|
+
_(Sneakers::ContentEncoding.decode(gzip_compress('foobar'), 'gzip')).must_equal('foobar')
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe '.encode' do
|
23
|
+
it 'uses the given encoder' do
|
24
|
+
Sneakers::ContentEncoding.register(
|
25
|
+
content_encoding: 'gzip',
|
26
|
+
encoder: ->(payload) { gzip_compress(payload) },
|
27
|
+
decoder: ->(_) {},
|
28
|
+
)
|
29
|
+
|
30
|
+
_(gzip_decompress(Sneakers::ContentEncoding.encode('foobar', 'gzip'))).must_equal('foobar')
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'passes the payload through by default' do
|
34
|
+
payload = "just some text"
|
35
|
+
_(Sneakers::ContentEncoding.encode(payload, 'unknown/encoding')).must_equal(payload)
|
36
|
+
_(Sneakers::ContentEncoding.decode(payload, 'unknown/encoding')).must_equal(payload)
|
37
|
+
_(Sneakers::ContentEncoding.encode(payload, nil)).must_equal(payload)
|
38
|
+
_(Sneakers::ContentEncoding.decode(payload, nil)).must_equal(payload)
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'passes the payload through if type not found' do
|
42
|
+
Sneakers::ContentEncoding.register(content_encoding: 'found/encoding', encoder: ->(_) {}, decoder: ->(_) {})
|
43
|
+
payload = "just some text"
|
44
|
+
|
45
|
+
_(Sneakers::ContentEncoding.encode(payload, 'unknown/encoding')).must_equal(payload)
|
46
|
+
_(Sneakers::ContentEncoding.decode(payload, 'unknown/encoding')).must_equal(payload)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
describe '.register' do
|
51
|
+
it 'provides a mechnism to register a given encoding' do
|
52
|
+
Sneakers::ContentEncoding.register(
|
53
|
+
content_encoding: 'gzip',
|
54
|
+
encoder: ->(payload) { gzip_compress(payload) },
|
55
|
+
decoder: ->(payload) { gzip_decompress(payload) },
|
56
|
+
)
|
57
|
+
|
58
|
+
ce = Sneakers::ContentEncoding
|
59
|
+
_(ce.decode(ce.encode('hello world', 'gzip'), 'gzip')).must_equal('hello world')
|
60
|
+
end
|
61
|
+
|
62
|
+
it 'requires a content encoding' do
|
63
|
+
_(proc { Sneakers::ContentEncoding.register(encoder: -> { }, decoder: -> { }) }).must_raise ArgumentError
|
64
|
+
end
|
65
|
+
|
66
|
+
it 'expects encoder and decoder to be present' do
|
67
|
+
_(proc { Sneakers::ContentEncoding.register(content_encoding: 'foo', decoder: -> { }) }).must_raise ArgumentError
|
68
|
+
_(proc { Sneakers::ContentEncoding.register(content_encoding: 'foo', encoder: -> { }) }).must_raise ArgumentError
|
69
|
+
end
|
70
|
+
|
71
|
+
it 'expects encoder and decoder to be a proc' do
|
72
|
+
_(proc { Sneakers::ContentEncoding.register(content_encoding: 'foo', encoder: 'not a proc', decoder: ->(_) { }) }).must_raise ArgumentError
|
73
|
+
_(proc { Sneakers::ContentEncoding.register(content_encoding: 'foo', encoder: ->(_) {}, decoder: 'not a proc' ) }).must_raise ArgumentError
|
74
|
+
end
|
75
|
+
|
76
|
+
it 'expects encoder and deseridecoderalizer to have the correct arity' do
|
77
|
+
_(proc { Sneakers::ContentEncoding.register(content_encoding: 'foo', encoder: ->(_,_) {}, decoder: ->(_) {}) }).must_raise ArgumentError
|
78
|
+
_(proc { Sneakers::ContentEncoding.register(content_encoding: 'foo', encoder: ->(_) {}, decoder: ->() {} ) }).must_raise ArgumentError
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sneakers/content_type'
|
3
|
+
require 'base64'
|
4
|
+
|
5
|
+
describe Sneakers::ContentType do
|
6
|
+
after do
|
7
|
+
Sneakers::ContentType.reset!
|
8
|
+
end
|
9
|
+
|
10
|
+
describe '.deserialize' do
|
11
|
+
it 'uses the given deserializer' do
|
12
|
+
Sneakers::ContentType.register(
|
13
|
+
content_type: 'application/json',
|
14
|
+
serializer: ->(_) {},
|
15
|
+
deserializer: ->(payload) { JSON.parse(payload) },
|
16
|
+
)
|
17
|
+
|
18
|
+
_(Sneakers::ContentType.deserialize('{"foo":"bar"}', 'application/json')).must_equal('foo' => 'bar')
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe '.serialize' do
|
23
|
+
it 'uses the given serializer' do
|
24
|
+
Sneakers::ContentType.register(
|
25
|
+
content_type: 'application/json',
|
26
|
+
serializer: ->(payload) { JSON.dump(payload) },
|
27
|
+
deserializer: ->(_) {},
|
28
|
+
)
|
29
|
+
|
30
|
+
_(Sneakers::ContentType.serialize({ 'foo' => 'bar' }, 'application/json')).must_equal('{"foo":"bar"}')
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'passes the payload through by default' do
|
34
|
+
payload = "just some text"
|
35
|
+
_(Sneakers::ContentType.serialize(payload, 'unknown/type')).must_equal(payload)
|
36
|
+
_(Sneakers::ContentType.deserialize(payload, 'unknown/type')).must_equal(payload)
|
37
|
+
_(Sneakers::ContentType.serialize(payload, nil)).must_equal(payload)
|
38
|
+
_(Sneakers::ContentType.deserialize(payload, nil)).must_equal(payload)
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'passes the payload through if type not found' do
|
42
|
+
Sneakers::ContentType.register(content_type: 'found/type', serializer: ->(_) {}, deserializer: ->(_) {})
|
43
|
+
payload = "just some text"
|
44
|
+
|
45
|
+
_(Sneakers::ContentType.serialize(payload, 'unknown/type')).must_equal(payload)
|
46
|
+
_(Sneakers::ContentType.deserialize(payload, 'unknown/type')).must_equal(payload)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
describe '.register' do
|
51
|
+
it 'provides a mechnism to register a given type' do
|
52
|
+
Sneakers::ContentType.register(
|
53
|
+
content_type: 'text/base64',
|
54
|
+
serializer: ->(payload) { Base64.encode64(payload) },
|
55
|
+
deserializer: ->(payload) { Base64.decode64(payload) },
|
56
|
+
)
|
57
|
+
|
58
|
+
ct = Sneakers::ContentType
|
59
|
+
_(ct.deserialize(ct.serialize('hello world', 'text/base64'), 'text/base64')).must_equal('hello world')
|
60
|
+
end
|
61
|
+
|
62
|
+
it 'requires a content type' do
|
63
|
+
_(proc { Sneakers::ContentType.register(serializer: -> { }, deserializer: -> { }) }).must_raise ArgumentError
|
64
|
+
end
|
65
|
+
|
66
|
+
it 'expects serializer and deserializer to be present' do
|
67
|
+
_(proc { Sneakers::ContentType.register(content_type: 'foo', deserializer: -> { }) }).must_raise ArgumentError
|
68
|
+
_(proc { Sneakers::ContentType.register(content_type: 'foo', serializer: -> { }) }).must_raise ArgumentError
|
69
|
+
end
|
70
|
+
|
71
|
+
it 'expects serializer and deserializer to be a proc' do
|
72
|
+
_(proc { Sneakers::ContentType.register(content_type: 'foo', serializer: 'not a proc', deserializer: ->(_) { }) }).must_raise ArgumentError
|
73
|
+
_(proc { Sneakers::ContentType.register(content_type: 'foo', serializer: ->(_) {}, deserializer: 'not a proc' ) }).must_raise ArgumentError
|
74
|
+
end
|
75
|
+
|
76
|
+
it 'expects serializer and deserializer to have the correct arity' do
|
77
|
+
_(proc { Sneakers::ContentType.register(content_type: 'foo', serializer: ->(_,_) {}, deserializer: ->(_) {}) }).must_raise ArgumentError
|
78
|
+
_(proc { Sneakers::ContentType.register(content_type: 'foo', serializer: ->(_) {}, deserializer: ->() {} ) }).must_raise ArgumentError
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,158 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sneakers'
|
3
|
+
require 'sneakers/runner'
|
4
|
+
require 'fixtures/integration_worker'
|
5
|
+
|
6
|
+
require "rabbitmq/http/client"
|
7
|
+
require 'timeout'
|
8
|
+
|
9
|
+
|
10
|
+
describe "integration" do
|
11
|
+
describe 'first' do
|
12
|
+
before :each do
|
13
|
+
skip unless ENV['INTEGRATION']
|
14
|
+
prepare
|
15
|
+
end
|
16
|
+
|
17
|
+
def integration_log(msg)
|
18
|
+
puts msg if ENV['INTEGRATION_LOG']
|
19
|
+
end
|
20
|
+
|
21
|
+
def rmq_addr
|
22
|
+
@rmq_addr ||= compose_or_localhost("rabbitmq")
|
23
|
+
end
|
24
|
+
|
25
|
+
def admin
|
26
|
+
@admin ||=
|
27
|
+
begin
|
28
|
+
puts "RABBITMQ is at #{rmq_addr}"
|
29
|
+
RabbitMQ::HTTP::Client.new("http://#{rmq_addr}:15672/", username: "guest", password: "guest")
|
30
|
+
rescue
|
31
|
+
fail "Rabbitmq admin seems to not exist? you better be running this on Travis or Docker. proceeding.\n#{$!}"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def prepare
|
36
|
+
# clean up all integration queues; admin interface must be installed
|
37
|
+
# in integration env
|
38
|
+
qs = admin.list_queues
|
39
|
+
qs.each do |q|
|
40
|
+
name = q.name
|
41
|
+
if name.start_with? 'integration_'
|
42
|
+
admin.delete_queue('/', name)
|
43
|
+
integration_log "cleaning up #{name}."
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
Sneakers.clear!
|
48
|
+
Sneakers.configure(:amqp => "amqp://guest:guest@#{rmq_addr}:5672")
|
49
|
+
Sneakers.logger.level = Logger::ERROR
|
50
|
+
|
51
|
+
# configure integration worker on a random generated queue
|
52
|
+
random_queue = "integration_#{rand(10**36).to_s(36)}"
|
53
|
+
|
54
|
+
redis_addr = compose_or_localhost("redis")
|
55
|
+
@redis = Redis.new(:host => redis_addr)
|
56
|
+
@redis.del(random_queue)
|
57
|
+
IntegrationWorker.from_queue(random_queue)
|
58
|
+
end
|
59
|
+
|
60
|
+
def assert_all_accounted_for(opts)
|
61
|
+
integration_log 'waiting for publishes to stabilize (5s).'
|
62
|
+
sleep 5
|
63
|
+
|
64
|
+
integration_log "polling for changes (max #{opts[:within_sec]}s)."
|
65
|
+
pid = opts[:pid]
|
66
|
+
opts[:within_sec].times do
|
67
|
+
sleep 1
|
68
|
+
count = @redis.get(opts[:queue]).to_i
|
69
|
+
if count == opts[:jobs]
|
70
|
+
integration_log "#{count} jobs accounted for successfully."
|
71
|
+
Process.kill("TERM", pid)
|
72
|
+
sleep 1
|
73
|
+
return
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
integration_log "failed test. killing off workers."
|
78
|
+
Process.kill("TERM", pid)
|
79
|
+
sleep 1
|
80
|
+
fail "incomplete!"
|
81
|
+
end
|
82
|
+
|
83
|
+
def start_worker(w)
|
84
|
+
integration_log "starting workers."
|
85
|
+
r = Sneakers::Runner.new([w])
|
86
|
+
pid = fork {
|
87
|
+
r.run
|
88
|
+
}
|
89
|
+
|
90
|
+
integration_log "waiting for workers to stabilize (5s)."
|
91
|
+
sleep 5
|
92
|
+
|
93
|
+
pid
|
94
|
+
end
|
95
|
+
|
96
|
+
def consumers_count
|
97
|
+
qs = admin.list_queues
|
98
|
+
qs.each do |q|
|
99
|
+
if q.name.start_with? 'integration_'
|
100
|
+
return [q.consumers, q.name]
|
101
|
+
end
|
102
|
+
end
|
103
|
+
return [0, nil]
|
104
|
+
end
|
105
|
+
|
106
|
+
def assert_any_consumers(consumers_should_be_there, maximum_wait_time = 15)
|
107
|
+
Timeout::timeout(maximum_wait_time) do
|
108
|
+
loop do
|
109
|
+
consumers, queue = consumers_count
|
110
|
+
fail 'no queues so no consumers' if consumers_should_be_there && !queue
|
111
|
+
puts "We see #{consumers} consumers on #{queue}"
|
112
|
+
(consumers_should_be_there == consumers.zero?) ? sleep(1) : return
|
113
|
+
end
|
114
|
+
end
|
115
|
+
rescue Timeout::Error
|
116
|
+
fail "Consumers should #{'not' unless consumers_should_be_there} be here but #{consumers} consumers were after #{maximum_wait_time}s waiting."
|
117
|
+
end
|
118
|
+
|
119
|
+
it 'should be possible to terminate when queue is full' do
|
120
|
+
job_count = 40000
|
121
|
+
|
122
|
+
pid = start_worker(IntegrationWorker)
|
123
|
+
Process.kill("TERM", pid)
|
124
|
+
|
125
|
+
integration_log "publishing #{job_count} messages..."
|
126
|
+
p = Sneakers::Publisher.new
|
127
|
+
job_count.times do |i|
|
128
|
+
p.publish("m #{i}", to_queue: IntegrationWorker.queue_name)
|
129
|
+
end
|
130
|
+
|
131
|
+
pid = start_worker(IntegrationWorker)
|
132
|
+
assert_any_consumers true
|
133
|
+
integration_log "Killing #{pid} now!"
|
134
|
+
Process.kill("TERM", pid)
|
135
|
+
assert_any_consumers false
|
136
|
+
end
|
137
|
+
|
138
|
+
it 'should pull down 100 jobs from a real queue' do
|
139
|
+
job_count = 100
|
140
|
+
|
141
|
+
pid = start_worker(IntegrationWorker)
|
142
|
+
|
143
|
+
integration_log "publishing..."
|
144
|
+
p = Sneakers::Publisher.new
|
145
|
+
job_count.times do |i|
|
146
|
+
p.publish("m #{i}", to_queue: IntegrationWorker.queue_name)
|
147
|
+
end
|
148
|
+
|
149
|
+
assert_all_accounted_for(
|
150
|
+
queue: IntegrationWorker.queue_name,
|
151
|
+
pid: pid,
|
152
|
+
within_sec: 15,
|
153
|
+
jobs: job_count,
|
154
|
+
)
|
155
|
+
end
|
156
|
+
|
157
|
+
end
|
158
|
+
end
|
@@ -0,0 +1,179 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'gzip_helper'
|
3
|
+
require 'sneakers'
|
4
|
+
require 'serverengine'
|
5
|
+
|
6
|
+
describe Sneakers::Publisher do
|
7
|
+
let :pub_vars do
|
8
|
+
{
|
9
|
+
:prefetch => 25,
|
10
|
+
:durable => true,
|
11
|
+
:ack => true,
|
12
|
+
:heartbeat => 2,
|
13
|
+
:vhost => '/',
|
14
|
+
:exchange => "sneakers",
|
15
|
+
:exchange_type => :direct,
|
16
|
+
:exchange_arguments => { 'x-arg' => 'value' }
|
17
|
+
}
|
18
|
+
end
|
19
|
+
|
20
|
+
describe '#publish' do
|
21
|
+
before do
|
22
|
+
Sneakers.clear!
|
23
|
+
Sneakers.configure(:log => 'sneakers.log')
|
24
|
+
end
|
25
|
+
|
26
|
+
it 'should publish a message to an exchange' do
|
27
|
+
xchg = Object.new
|
28
|
+
mock(xchg).publish('test msg', { routing_key: 'downloads' })
|
29
|
+
|
30
|
+
p = Sneakers::Publisher.new
|
31
|
+
p.instance_variable_set(:@exchange, xchg)
|
32
|
+
|
33
|
+
mock(p).ensure_connection! {}
|
34
|
+
p.publish('test msg', to_queue: 'downloads')
|
35
|
+
end
|
36
|
+
|
37
|
+
it 'should publish with the persistence specified' do
|
38
|
+
xchg = Object.new
|
39
|
+
mock(xchg).publish('test msg', { routing_key: 'downloads', persistence: true })
|
40
|
+
|
41
|
+
p = Sneakers::Publisher.new
|
42
|
+
p.instance_variable_set(:@exchange, xchg)
|
43
|
+
|
44
|
+
mock(p).ensure_connection! {}
|
45
|
+
p.publish('test msg', to_queue: 'downloads', persistence: true)
|
46
|
+
end
|
47
|
+
|
48
|
+
it 'should publish with arbitrary metadata specified' do
|
49
|
+
xchg = Object.new
|
50
|
+
mock(xchg).publish('test msg', { routing_key: 'downloads', expiration: 1, headers: {foo: 'bar'} })
|
51
|
+
|
52
|
+
p = Sneakers::Publisher.new
|
53
|
+
p.instance_variable_set(:@exchange, xchg)
|
54
|
+
|
55
|
+
mock(p).ensure_connection! {}
|
56
|
+
p.publish('test msg', to_queue: 'downloads', expiration: 1, headers: {foo: 'bar'})
|
57
|
+
end
|
58
|
+
|
59
|
+
it 'should not reconnect if already connected' do
|
60
|
+
xchg = Object.new
|
61
|
+
mock(xchg).publish('test msg', { routing_key: 'downloads' })
|
62
|
+
|
63
|
+
p = Sneakers::Publisher.new
|
64
|
+
p.instance_variable_set(:@exchange, xchg)
|
65
|
+
|
66
|
+
mock(p).connected? { true }
|
67
|
+
mock(p).connect!.times(0)
|
68
|
+
|
69
|
+
p.publish('test msg', to_queue: 'downloads')
|
70
|
+
end
|
71
|
+
|
72
|
+
it 'should connect to rabbitmq configured on Sneakers.configure' do
|
73
|
+
logger = Logger.new('/dev/null')
|
74
|
+
Sneakers.configure(
|
75
|
+
amqp: 'amqp://someuser:somepassword@somehost:5672',
|
76
|
+
heartbeat: 1,
|
77
|
+
exchange: 'another_exchange',
|
78
|
+
exchange_options: { :type => :topic, :arguments => { 'x-arg' => 'value' } },
|
79
|
+
log: logger,
|
80
|
+
properties: { key: "value" },
|
81
|
+
durable: false)
|
82
|
+
|
83
|
+
channel = Object.new
|
84
|
+
mock(channel).exchange('another_exchange', type: :topic, durable: false, :auto_delete => false, arguments: { 'x-arg' => 'value' }) do
|
85
|
+
mock(Object.new).publish('test msg', { routing_key: 'downloads' })
|
86
|
+
end
|
87
|
+
|
88
|
+
bunny = Object.new
|
89
|
+
mock(bunny).start
|
90
|
+
mock(bunny).create_channel { channel }
|
91
|
+
|
92
|
+
mock(Bunny).new('amqp://someuser:somepassword@somehost:5672', heartbeat: 1, vhost: '/', logger: logger, properties: { key: "value" }) { bunny }
|
93
|
+
|
94
|
+
p = Sneakers::Publisher.new
|
95
|
+
|
96
|
+
p.publish('test msg', to_queue: 'downloads')
|
97
|
+
end
|
98
|
+
|
99
|
+
describe 'externally instantiated bunny session' do
|
100
|
+
let(:my_vars) { pub_vars.merge(to_queue: 'downloads') }
|
101
|
+
before do
|
102
|
+
logger = Logger.new('/dev/null')
|
103
|
+
channel = Object.new
|
104
|
+
exchange = Object.new
|
105
|
+
existing_session = Bunny.new
|
106
|
+
|
107
|
+
mock(existing_session).start
|
108
|
+
mock(existing_session).create_channel { channel }
|
109
|
+
|
110
|
+
mock(channel).exchange('another_exchange', type: :topic, durable: false, :auto_delete => false, arguments: { 'x-arg' => 'value' }) do
|
111
|
+
exchange
|
112
|
+
end
|
113
|
+
|
114
|
+
mock(exchange).publish('test msg', my_vars)
|
115
|
+
|
116
|
+
Sneakers.configure(
|
117
|
+
connection: existing_session,
|
118
|
+
heartbeat: 1, exchange: 'another_exchange',
|
119
|
+
exchange_type: :topic,
|
120
|
+
exchange_arguments: { 'x-arg' => 'value' },
|
121
|
+
log: logger,
|
122
|
+
durable: false
|
123
|
+
)
|
124
|
+
@existing_session = existing_session
|
125
|
+
end
|
126
|
+
|
127
|
+
it 'can handle an existing connection that is offline' do
|
128
|
+
p = Sneakers::Publisher.new
|
129
|
+
p.publish('test msg', my_vars)
|
130
|
+
_(p.instance_variable_get(:@bunny)).must_equal @existing_session
|
131
|
+
end
|
132
|
+
|
133
|
+
it 'can handle an existing connection that is online' do
|
134
|
+
mock(@existing_session).connected? { true }
|
135
|
+
p = Sneakers::Publisher.new
|
136
|
+
p.publish('test msg', my_vars)
|
137
|
+
_(p.instance_variable_get(:@bunny)).must_equal @existing_session
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
it 'should publish using the content type serializer' do
|
142
|
+
Sneakers::ContentType.register(
|
143
|
+
content_type: 'application/json',
|
144
|
+
serializer: ->(payload) { JSON.dump(payload) },
|
145
|
+
deserializer: ->(_) {},
|
146
|
+
)
|
147
|
+
|
148
|
+
xchg = Object.new
|
149
|
+
mock(xchg).publish('{"foo":"bar"}', { routing_key: 'downloads', content_type: 'application/json' })
|
150
|
+
|
151
|
+
p = Sneakers::Publisher.new
|
152
|
+
p.instance_variable_set(:@exchange, xchg)
|
153
|
+
|
154
|
+
mock(p).ensure_connection! {}
|
155
|
+
p.publish({ 'foo' => 'bar' }, to_queue: 'downloads', content_type: 'application/json')
|
156
|
+
|
157
|
+
Sneakers::ContentType.reset!
|
158
|
+
end
|
159
|
+
|
160
|
+
it 'should publish using the content encoding encoder' do
|
161
|
+
Sneakers::ContentEncoding.register(
|
162
|
+
content_encoding: 'gzip',
|
163
|
+
encoder: ->(payload) { gzip_compress(payload) },
|
164
|
+
decoder: ->(_) {},
|
165
|
+
)
|
166
|
+
|
167
|
+
xchg = Object.new
|
168
|
+
mock(xchg).publish(gzip_compress('foobar'), { routing_key: 'downloads', content_encoding: 'gzip' })
|
169
|
+
|
170
|
+
p = Sneakers::Publisher.new
|
171
|
+
p.instance_variable_set(:@exchange, xchg)
|
172
|
+
|
173
|
+
mock(p).ensure_connection! {}
|
174
|
+
p.publish('foobar', to_queue: 'downloads', content_encoding: 'gzip')
|
175
|
+
|
176
|
+
Sneakers::ContentEncoding.reset!
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|