sneakers 2.5.0 → 2.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +4 -2
- data/examples/newrelic_metrics_worker.rb +2 -7
- data/lib/sneakers.rb +3 -2
- data/lib/sneakers/content_type.rb +47 -0
- data/lib/sneakers/handlers/maxretry.rb +1 -1
- data/lib/sneakers/publisher.rb +5 -3
- data/lib/sneakers/queue.rb +17 -4
- data/lib/sneakers/spawner.rb +1 -0
- data/lib/sneakers/version.rb +1 -1
- data/lib/sneakers/worker.rb +12 -8
- data/lib/sneakers/workergroup.rb +16 -2
- data/sneakers.gemspec +3 -2
- data/spec/sneakers/content_type_spec.rb +81 -0
- data/spec/sneakers/integration_spec.rb +42 -4
- data/spec/sneakers/publisher_spec.rb +22 -1
- data/spec/sneakers/worker_handlers_spec.rb +1 -6
- data/spec/sneakers/worker_spec.rb +75 -6
- data/spec/spec_helper.rb +1 -1
- metadata +13 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c52a398aecf272a36fcb3431f7b242a0d36dc0d1
|
4
|
+
data.tar.gz: c5974b0f7823f4d1e91c99d88adc3bbe50a2e572
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 85bc282b42260ebb6a08fdab01fc2396a30450f3cabb93bed27ccac801bc216e5e139613349cfb874a9c5c2a453234591759671de64c9f7654ddd5076384f746
|
7
|
+
data.tar.gz: 06fd1978121e6e59971152fd202e1cb2d5b13225d0f08f75583a5d021b0cfb8d52768102a007cef7d1dc191ed9264505a9e4157987054bc9631ee3eaaf749605
|
data/.travis.yml
CHANGED
@@ -25,8 +25,8 @@ class MetricsWorker
|
|
25
25
|
logger.info "FOUND <#{title}>"
|
26
26
|
ack!
|
27
27
|
end
|
28
|
-
|
29
|
-
add_transaction_tracer :work, name: 'MetricsWorker', params: 'args[0]'
|
28
|
+
|
29
|
+
add_transaction_tracer :work, name: 'MetricsWorker', params: 'args[0]', category: :task
|
30
30
|
|
31
31
|
private
|
32
32
|
|
@@ -36,10 +36,5 @@ class MetricsWorker
|
|
36
36
|
end
|
37
37
|
end
|
38
38
|
|
39
|
-
|
40
39
|
r = Sneakers::Runner.new([ MetricsWorker ])
|
41
40
|
r.run
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
data/lib/sneakers.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
|
-
require
|
2
|
-
require '
|
1
|
+
require 'sneakers/version'
|
2
|
+
require 'concurrent/executors'
|
3
3
|
require 'bunny'
|
4
4
|
require 'logger'
|
5
5
|
require 'serverengine'
|
@@ -17,6 +17,7 @@ require 'sneakers/support/production_formatter'
|
|
17
17
|
require 'sneakers/concerns/logging'
|
18
18
|
require 'sneakers/concerns/metrics'
|
19
19
|
require 'sneakers/handlers/oneshot'
|
20
|
+
require 'sneakers/content_type'
|
20
21
|
require 'sneakers/worker'
|
21
22
|
require 'sneakers/publisher'
|
22
23
|
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module Sneakers
|
2
|
+
class ContentType
|
3
|
+
def self.register(content_type: nil, serializer: nil, deserializer: nil)
|
4
|
+
# This can be removed when support is dropped for ruby 2.0 and replaced
|
5
|
+
# by a keyword arg with no default value
|
6
|
+
fail ArgumentError, 'missing keyword: content_type' if content_type.nil?
|
7
|
+
fail ArgumentError, 'missing keyword: serializer' if serializer.nil?
|
8
|
+
fail ArgumentError, 'missing keyword: deserializer' if deserializer.nil?
|
9
|
+
|
10
|
+
fail ArgumentError, "#{content_type} serializer must be a proc" unless serializer.is_a? Proc
|
11
|
+
fail ArgumentError, "#{content_type} deserializer must be a proc" unless deserializer.is_a? Proc
|
12
|
+
|
13
|
+
fail ArgumentError, "#{content_type} serializer must accept one argument, the payload" unless serializer.arity == 1
|
14
|
+
fail ArgumentError, "#{content_type} deserializer must accept one argument, the payload" unless deserializer.arity == 1
|
15
|
+
@_types[content_type] = new(serializer, deserializer)
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.serialize(payload, content_type)
|
19
|
+
return payload unless content_type
|
20
|
+
@_types[content_type].serializer.(payload)
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.deserialize(payload, content_type)
|
24
|
+
return payload unless content_type
|
25
|
+
@_types[content_type].deserializer.(payload)
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.reset!
|
29
|
+
@_types = Hash.new(
|
30
|
+
new(passthrough, passthrough)
|
31
|
+
)
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.passthrough
|
35
|
+
->(payload) { payload }
|
36
|
+
end
|
37
|
+
|
38
|
+
def initialize(serializer, deserializer)
|
39
|
+
@serializer = serializer
|
40
|
+
@deserializer = deserializer
|
41
|
+
end
|
42
|
+
|
43
|
+
attr_reader :serializer, :deserializer
|
44
|
+
|
45
|
+
reset!
|
46
|
+
end
|
47
|
+
end
|
@@ -135,7 +135,7 @@ module Sneakers
|
|
135
135
|
"#{log_prefix} msg=failing, retry_count=#{num_attempts}, reason=#{reason}"
|
136
136
|
end
|
137
137
|
data = {
|
138
|
-
error: reason,
|
138
|
+
error: reason.to_s,
|
139
139
|
num_attempts: num_attempts,
|
140
140
|
failed_at: Time.now.iso8601,
|
141
141
|
payload: Base64.encode64(msg.to_s),
|
data/lib/sneakers/publisher.rb
CHANGED
@@ -12,7 +12,7 @@ module Sneakers
|
|
12
12
|
to_queue = options.delete(:to_queue)
|
13
13
|
options[:routing_key] ||= to_queue
|
14
14
|
Sneakers.logger.info {"publishing <#{msg}> to [#{options[:routing_key]}]"}
|
15
|
-
@exchange.publish(msg, options)
|
15
|
+
@exchange.publish(ContentType.serialize(msg, options[:content_type]), options)
|
16
16
|
end
|
17
17
|
|
18
18
|
|
@@ -34,8 +34,10 @@ module Sneakers
|
|
34
34
|
end
|
35
35
|
|
36
36
|
def create_bunny_connection
|
37
|
-
Bunny.new(@opts[:amqp], :vhost => @opts[:vhost],
|
37
|
+
Bunny.new(@opts[:amqp], :vhost => @opts[:vhost],
|
38
|
+
:heartbeat => @opts[:heartbeat],
|
39
|
+
:properties => @opts.fetch(:properties, {}),
|
40
|
+
:logger => Sneakers::logger)
|
38
41
|
end
|
39
42
|
end
|
40
43
|
end
|
41
|
-
|
data/lib/sneakers/queue.rb
CHANGED
@@ -56,13 +56,26 @@ class Sneakers::Queue
|
|
56
56
|
end
|
57
57
|
|
58
58
|
def unsubscribe
|
59
|
-
|
60
|
-
|
61
|
-
|
59
|
+
return unless @consumer
|
60
|
+
|
61
|
+
# TODO: should we simply close the channel here?
|
62
|
+
Sneakers.logger.info("Queue: will try to cancel consumer #{@consumer.inspect}")
|
63
|
+
cancel_ok = @consumer.cancel
|
64
|
+
if cancel_ok
|
65
|
+
Sneakers.logger.info "Queue: consumer #{cancel_ok.consumer_tag} cancelled"
|
66
|
+
@consumer = nil
|
67
|
+
else
|
68
|
+
Sneakers.logger.warn "Queue: could not cancel consumer #{@consumer.inspect}"
|
69
|
+
sleep(1)
|
70
|
+
unsubscribe
|
71
|
+
end
|
62
72
|
end
|
63
73
|
|
64
74
|
def create_bunny_connection
|
65
|
-
Bunny.new(@opts[:amqp], :vhost => @opts[:vhost],
|
75
|
+
Bunny.new(@opts[:amqp], :vhost => @opts[:vhost],
|
76
|
+
:heartbeat => @opts[:heartbeat],
|
77
|
+
:properties => @opts.fetch(:properties, {}),
|
78
|
+
:logger => Sneakers::logger)
|
66
79
|
end
|
67
80
|
private :create_bunny_connection
|
68
81
|
end
|
data/lib/sneakers/spawner.rb
CHANGED
@@ -8,6 +8,7 @@ module Sneakers
|
|
8
8
|
unless File.exists?(worker_group_config_file)
|
9
9
|
puts "No worker group file found."
|
10
10
|
puts "Specify via ENV 'WORKER_GROUP_CONFIG' or by convention ./config/sneaker_worker_groups.yml"
|
11
|
+
Kernel.exit(1)
|
11
12
|
end
|
12
13
|
@pids = []
|
13
14
|
@exec_string = "bundle exec rake sneakers:run"
|
data/lib/sneakers/version.rb
CHANGED
data/lib/sneakers/worker.rb
CHANGED
@@ -19,8 +19,9 @@ module Sneakers
|
|
19
19
|
|
20
20
|
@should_ack = opts[:ack]
|
21
21
|
@timeout_after = opts[:timeout_job_after]
|
22
|
-
@pool = pool ||
|
22
|
+
@pool = pool || Concurrent::FixedThreadPool.new(opts[:threads])
|
23
23
|
@call_with_params = respond_to?(:work_with_params)
|
24
|
+
@content_type = opts[:content_type]
|
24
25
|
|
25
26
|
@queue = queue || Sneakers::Queue.new(
|
26
27
|
queue_name,
|
@@ -39,13 +40,13 @@ module Sneakers
|
|
39
40
|
to_queue = opts.delete(:to_queue)
|
40
41
|
opts[:routing_key] ||= to_queue
|
41
42
|
return unless opts[:routing_key]
|
42
|
-
@queue.exchange.publish(msg, opts)
|
43
|
+
@queue.exchange.publish(Sneakers::ContentType.serialize(msg, opts[:content_type]), opts)
|
43
44
|
end
|
44
45
|
|
45
46
|
def do_work(delivery_info, metadata, msg, handler)
|
46
47
|
worker_trace "Working off: #{msg.inspect}"
|
47
48
|
|
48
|
-
@pool.
|
49
|
+
@pool.post do
|
49
50
|
res = nil
|
50
51
|
error = nil
|
51
52
|
|
@@ -53,10 +54,11 @@ module Sneakers
|
|
53
54
|
metrics.increment("work.#{self.class.name}.started")
|
54
55
|
Timeout.timeout(@timeout_after, WorkerTimeout) do
|
55
56
|
metrics.timing("work.#{self.class.name}.time") do
|
57
|
+
deserialized_msg = ContentType.deserialize(msg, @content_type || metadata && metadata[:content_type])
|
56
58
|
if @call_with_params
|
57
|
-
res = work_with_params(
|
59
|
+
res = work_with_params(deserialized_msg, delivery_info, metadata)
|
58
60
|
else
|
59
|
-
res = work(
|
61
|
+
res = work(deserialized_msg)
|
60
62
|
end
|
61
63
|
end
|
62
64
|
end
|
@@ -91,14 +93,15 @@ module Sneakers
|
|
91
93
|
end
|
92
94
|
|
93
95
|
metrics.increment("work.#{self.class.name}.ended")
|
94
|
-
end #
|
96
|
+
end #post
|
95
97
|
end
|
96
98
|
|
97
99
|
def stop
|
98
|
-
worker_trace "Stopping worker: shutting down thread pool."
|
99
|
-
@pool.shutdown
|
100
100
|
worker_trace "Stopping worker: unsubscribing."
|
101
101
|
@queue.unsubscribe
|
102
|
+
worker_trace "Stopping worker: shutting down thread pool."
|
103
|
+
@pool.shutdown
|
104
|
+
@pool.wait_for_termination
|
102
105
|
worker_trace "Stopping worker: I'm gone."
|
103
106
|
end
|
104
107
|
|
@@ -135,6 +138,7 @@ module Sneakers
|
|
135
138
|
|
136
139
|
def enqueue(msg, opts={})
|
137
140
|
opts[:routing_key] ||= @queue_opts[:routing_key]
|
141
|
+
opts[:content_type] ||= @queue_opts[:content_type]
|
138
142
|
opts[:to_queue] ||= @queue_name
|
139
143
|
|
140
144
|
publisher.publish(msg, opts)
|
data/lib/sneakers/workergroup.rb
CHANGED
@@ -21,7 +21,7 @@ module Sneakers
|
|
21
21
|
|
22
22
|
# Allocate single thread pool if share_threads is set. This improves load balancing
|
23
23
|
# when used with many workers.
|
24
|
-
pool = config[:share_threads] ?
|
24
|
+
pool = config[:share_threads] ? Concurrent::FixedThreadPool.new(config[:threads]) : nil
|
25
25
|
|
26
26
|
worker_classes = config[:worker_classes]
|
27
27
|
|
@@ -29,7 +29,13 @@ module Sneakers
|
|
29
29
|
worker_classes = worker_classes.call
|
30
30
|
end
|
31
31
|
|
32
|
-
|
32
|
+
# if we don't provide a connection to a worker,
|
33
|
+
# the queue used in the worker will create a new one
|
34
|
+
# so if we want to have a shared bunny connection for the workers
|
35
|
+
# we must create it here
|
36
|
+
bunny_connection = create_connection_or_nil
|
37
|
+
|
38
|
+
@workers = worker_classes.map{|w| w.new(nil, pool, {connection: bunny_connection}) }
|
33
39
|
# if more than one worker this should be per worker
|
34
40
|
# accumulate clients and consumers as well
|
35
41
|
@workers.each do |worker|
|
@@ -52,5 +58,13 @@ module Sneakers
|
|
52
58
|
@stop_flag.set!
|
53
59
|
end
|
54
60
|
|
61
|
+
def create_bunny_connection
|
62
|
+
Bunny.new(Sneakers::CONFIG[:amqp], :vhost => Sneakers::CONFIG[:vhost], :heartbeat => Sneakers::CONFIG[:heartbeat], :logger => Sneakers::logger)
|
63
|
+
end
|
64
|
+
|
65
|
+
def create_connection_or_nil
|
66
|
+
config[:share_bunny_connection] ? create_bunny_connection : nil
|
67
|
+
end
|
68
|
+
private :create_bunny_connection, :create_connection_or_nil
|
55
69
|
end
|
56
70
|
end
|
data/sneakers.gemspec
CHANGED
@@ -11,6 +11,7 @@ Gem::Specification.new do |gem|
|
|
11
11
|
gem.description = %q( Fast background processing framework for Ruby and RabbitMQ )
|
12
12
|
gem.summary = %q( Fast background processing framework for Ruby and RabbitMQ )
|
13
13
|
gem.homepage = 'http://sneakers.io'
|
14
|
+
gem.license = 'MIT'
|
14
15
|
gem.required_ruby_version = Gem::Requirement.new(">= 2.0")
|
15
16
|
|
16
17
|
gem.files = `git ls-files`.split($/).reject { |f| f == 'Gemfile.lock' }
|
@@ -18,8 +19,8 @@ Gem::Specification.new do |gem|
|
|
18
19
|
gem.test_files = gem.files.grep(/^(test|spec|features)\//)
|
19
20
|
gem.require_paths = ['lib']
|
20
21
|
gem.add_dependency 'serverengine', '~> 1.5.11'
|
21
|
-
gem.add_dependency 'bunny', '~> 2.
|
22
|
-
gem.add_dependency '
|
22
|
+
gem.add_dependency 'bunny', '~> 2.7.0'
|
23
|
+
gem.add_dependency 'concurrent-ruby', '~> 1.0'
|
23
24
|
gem.add_dependency 'thor'
|
24
25
|
|
25
26
|
# for integration environment (see .travis.yml and integration_spec)
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sneakers/content_type'
|
3
|
+
require 'base64'
|
4
|
+
|
5
|
+
describe Sneakers::ContentType do
|
6
|
+
after do
|
7
|
+
Sneakers::ContentType.reset!
|
8
|
+
end
|
9
|
+
|
10
|
+
describe '.deserialize' do
|
11
|
+
it 'uses the given deserializer' do
|
12
|
+
Sneakers::ContentType.register(
|
13
|
+
content_type: 'application/json',
|
14
|
+
serializer: ->(_) {},
|
15
|
+
deserializer: ->(payload) { JSON.parse(payload) },
|
16
|
+
)
|
17
|
+
|
18
|
+
Sneakers::ContentType.deserialize('{"foo":"bar"}', 'application/json').must_equal('foo' => 'bar')
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe '.serialize' do
|
23
|
+
it 'uses the given serializer' do
|
24
|
+
Sneakers::ContentType.register(
|
25
|
+
content_type: 'application/json',
|
26
|
+
serializer: ->(payload) { JSON.dump(payload) },
|
27
|
+
deserializer: ->(_) {},
|
28
|
+
)
|
29
|
+
|
30
|
+
Sneakers::ContentType.serialize({ 'foo' => 'bar' }, 'application/json').must_equal('{"foo":"bar"}')
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'passes the payload through by default' do
|
34
|
+
payload = "just some text"
|
35
|
+
Sneakers::ContentType.serialize(payload, 'unknown/type').must_equal(payload)
|
36
|
+
Sneakers::ContentType.deserialize(payload, 'unknown/type').must_equal(payload)
|
37
|
+
Sneakers::ContentType.serialize(payload, nil).must_equal(payload)
|
38
|
+
Sneakers::ContentType.deserialize(payload, nil).must_equal(payload)
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'passes the payload through if type not found' do
|
42
|
+
Sneakers::ContentType.register(content_type: 'found/type', serializer: ->(_) {}, deserializer: ->(_) {})
|
43
|
+
payload = "just some text"
|
44
|
+
|
45
|
+
Sneakers::ContentType.serialize(payload, 'unknown/type').must_equal(payload)
|
46
|
+
Sneakers::ContentType.deserialize(payload, 'unknown/type').must_equal(payload)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
describe '.register' do
|
51
|
+
it 'provides a mechnism to register a given type' do
|
52
|
+
Sneakers::ContentType.register(
|
53
|
+
content_type: 'text/base64',
|
54
|
+
serializer: ->(payload) { Base64.encode64(payload) },
|
55
|
+
deserializer: ->(payload) { Base64.decode64(payload) },
|
56
|
+
)
|
57
|
+
|
58
|
+
ct = Sneakers::ContentType
|
59
|
+
ct.deserialize(ct.serialize('hello world', 'text/base64'), 'text/base64').must_equal('hello world')
|
60
|
+
end
|
61
|
+
|
62
|
+
it 'requires a content type' do
|
63
|
+
proc { Sneakers::ContentType.register(serializer: -> { }, deserializer: -> { }) }.must_raise ArgumentError
|
64
|
+
end
|
65
|
+
|
66
|
+
it 'expects serializer and deserializer to be present' do
|
67
|
+
proc { Sneakers::ContentType.register(content_type: 'foo', deserializer: -> { }) }.must_raise ArgumentError
|
68
|
+
proc { Sneakers::ContentType.register(content_type: 'foo', serializer: -> { }) }.must_raise ArgumentError
|
69
|
+
end
|
70
|
+
|
71
|
+
it 'expects serializer and deserializer to be a proc' do
|
72
|
+
proc { Sneakers::ContentType.register(content_type: 'foo', serializer: 'not a proc', deserializer: ->(_) { }) }.must_raise ArgumentError
|
73
|
+
proc { Sneakers::ContentType.register(content_type: 'foo', serializer: ->(_) {}, deserializer: 'not a proc' ) }.must_raise ArgumentError
|
74
|
+
end
|
75
|
+
|
76
|
+
it 'expects serializer and deserializer to have the correct arity' do
|
77
|
+
proc { Sneakers::ContentType.register(content_type: 'foo', serializer: ->(_,_) {}, deserializer: ->(_) {}) }.must_raise ArgumentError
|
78
|
+
proc { Sneakers::ContentType.register(content_type: 'foo', serializer: ->(_) {}, deserializer: ->() {} ) }.must_raise ArgumentError
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -65,7 +65,7 @@ describe "integration" do
|
|
65
65
|
return
|
66
66
|
end
|
67
67
|
end
|
68
|
-
|
68
|
+
|
69
69
|
integration_log "failed test. killing off workers."
|
70
70
|
Process.kill("TERM", pid)
|
71
71
|
sleep 1
|
@@ -85,6 +85,44 @@ describe "integration" do
|
|
85
85
|
pid
|
86
86
|
end
|
87
87
|
|
88
|
+
def any_consumers
|
89
|
+
rmq_addr = compose_or_localhost("rabbitmq")
|
90
|
+
result = false
|
91
|
+
begin
|
92
|
+
admin = RabbitMQ::HTTP::Client.new("http://#{rmq_addr}:15672/", username: "guest", password: "guest")
|
93
|
+
qs = admin.list_queues
|
94
|
+
qs.each do |q|
|
95
|
+
if q.name.start_with? 'integration_'
|
96
|
+
puts "We see #{q.consumers} consumers on #{q.name}"
|
97
|
+
return true if q.consumers > 0
|
98
|
+
end
|
99
|
+
end
|
100
|
+
return false
|
101
|
+
rescue
|
102
|
+
puts "Rabbitmq admin seems to not exist? you better be running this on Travis or Docker. proceeding.\n#{$!}"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
it 'should be possible to terminate when queue is full' do
|
107
|
+
job_count = 40000
|
108
|
+
|
109
|
+
pid = start_worker(IntegrationWorker)
|
110
|
+
Process.kill("TERM", pid)
|
111
|
+
|
112
|
+
integration_log "publishing #{job_count} messages..."
|
113
|
+
p = Sneakers::Publisher.new
|
114
|
+
job_count.times do |i|
|
115
|
+
p.publish("m #{i}", to_queue: IntegrationWorker.queue_name)
|
116
|
+
end
|
117
|
+
|
118
|
+
pid = start_worker(IntegrationWorker)
|
119
|
+
any_consumers.must_equal true
|
120
|
+
integration_log "Killing #{pid} now!"
|
121
|
+
Process.kill("TERM", pid)
|
122
|
+
sleep(2)
|
123
|
+
any_consumers.must_equal false
|
124
|
+
end
|
125
|
+
|
88
126
|
it 'should pull down 100 jobs from a real queue' do
|
89
127
|
job_count = 100
|
90
128
|
|
@@ -95,7 +133,7 @@ describe "integration" do
|
|
95
133
|
job_count.times do |i|
|
96
134
|
p.publish("m #{i}", to_queue: IntegrationWorker.queue_name)
|
97
135
|
end
|
98
|
-
|
136
|
+
|
99
137
|
assert_all_accounted_for(
|
100
138
|
queue: IntegrationWorker.queue_name,
|
101
139
|
pid: pid,
|
@@ -103,8 +141,8 @@ describe "integration" do
|
|
103
141
|
jobs: job_count,
|
104
142
|
)
|
105
143
|
end
|
106
|
-
|
144
|
+
|
107
145
|
end
|
108
146
|
end
|
109
|
-
|
147
|
+
|
110
148
|
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
require 'sneakers'
|
3
|
+
require 'serverengine'
|
3
4
|
|
4
5
|
describe Sneakers::Publisher do
|
5
6
|
let :pub_vars do
|
@@ -75,6 +76,7 @@ describe Sneakers::Publisher do
|
|
75
76
|
exchange: 'another_exchange',
|
76
77
|
exchange_options: { :type => :topic, :arguments => { 'x-arg' => 'value' } },
|
77
78
|
log: logger,
|
79
|
+
properties: { key: "value" },
|
78
80
|
durable: false)
|
79
81
|
|
80
82
|
channel = Object.new
|
@@ -86,7 +88,7 @@ describe Sneakers::Publisher do
|
|
86
88
|
mock(bunny).start
|
87
89
|
mock(bunny).create_channel { channel }
|
88
90
|
|
89
|
-
mock(Bunny).new('amqp://someuser:somepassword@somehost:5672', heartbeat: 1, vhost: '/', logger: logger) { bunny }
|
91
|
+
mock(Bunny).new('amqp://someuser:somepassword@somehost:5672', heartbeat: 1, vhost: '/', logger: logger, properties: { key: "value" }) { bunny }
|
90
92
|
|
91
93
|
p = Sneakers::Publisher.new
|
92
94
|
|
@@ -122,5 +124,24 @@ describe Sneakers::Publisher do
|
|
122
124
|
p.publish('test msg', my_vars)
|
123
125
|
p.instance_variable_get(:@bunny).must_equal existing_session
|
124
126
|
end
|
127
|
+
|
128
|
+
it 'should publish using the content type serializer' do
|
129
|
+
Sneakers::ContentType.register(
|
130
|
+
content_type: 'application/json',
|
131
|
+
serializer: ->(payload) { JSON.dump(payload) },
|
132
|
+
deserializer: ->(_) {},
|
133
|
+
)
|
134
|
+
|
135
|
+
xchg = Object.new
|
136
|
+
mock(xchg).publish('{"foo":"bar"}', routing_key: 'downloads', content_type: 'application/json')
|
137
|
+
|
138
|
+
p = Sneakers::Publisher.new
|
139
|
+
p.instance_variable_set(:@exchange, xchg)
|
140
|
+
|
141
|
+
mock(p).ensure_connection! {}
|
142
|
+
p.publish({ 'foo' => 'bar' }, to_queue: 'downloads', content_type: 'application/json')
|
143
|
+
|
144
|
+
Sneakers::ContentType.reset!
|
145
|
+
end
|
125
146
|
end
|
126
147
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
require 'sneakers'
|
3
3
|
require 'timeout'
|
4
|
+
require 'serverengine'
|
4
5
|
|
5
6
|
class DummyWorker
|
6
7
|
include Sneakers::Worker
|
@@ -75,7 +76,16 @@ class PublishingWorker
|
|
75
76
|
end
|
76
77
|
end
|
77
78
|
|
79
|
+
class JSONPublishingWorker
|
80
|
+
include Sneakers::Worker
|
81
|
+
from_queue 'defaults',
|
82
|
+
:ack => false,
|
83
|
+
:exchange => 'foochange'
|
78
84
|
|
85
|
+
def work(msg)
|
86
|
+
publish msg, :to_queue => 'target', :content_type => 'application/json'
|
87
|
+
end
|
88
|
+
end
|
79
89
|
|
80
90
|
class LoggingWorker
|
81
91
|
include Sneakers::Worker
|
@@ -87,6 +97,15 @@ class LoggingWorker
|
|
87
97
|
end
|
88
98
|
end
|
89
99
|
|
100
|
+
class JSONWorker
|
101
|
+
include Sneakers::Worker
|
102
|
+
from_queue 'defaults',
|
103
|
+
:ack => false,
|
104
|
+
:content_type => 'application/json'
|
105
|
+
|
106
|
+
def work(msg)
|
107
|
+
end
|
108
|
+
end
|
90
109
|
|
91
110
|
class MetricsWorker
|
92
111
|
include Sneakers::Worker
|
@@ -123,11 +142,7 @@ class WithDeprecatedExchangeOptionsWorker
|
|
123
142
|
end
|
124
143
|
end
|
125
144
|
|
126
|
-
|
127
|
-
def process(*args,&block)
|
128
|
-
block.call
|
129
|
-
end
|
130
|
-
end
|
145
|
+
TestPool ||= Concurrent::ImmediateExecutor
|
131
146
|
|
132
147
|
def with_test_queuefactory(ctx, ack=true, msg=nil, nowork=false)
|
133
148
|
qf = Object.new
|
@@ -164,7 +179,8 @@ describe Sneakers::Worker do
|
|
164
179
|
mock(Sneakers::Publisher).new(DummyWorker.queue_opts) do
|
165
180
|
mock(Object.new).publish(message, {
|
166
181
|
:routing_key => 'test.routing.key',
|
167
|
-
:to_queue => 'downloads'
|
182
|
+
:to_queue => 'downloads',
|
183
|
+
:content_type => nil,
|
168
184
|
})
|
169
185
|
end
|
170
186
|
|
@@ -330,6 +346,38 @@ describe Sneakers::Worker do
|
|
330
346
|
w.do_work(nil, nil, "msg", nil)
|
331
347
|
end
|
332
348
|
|
349
|
+
describe 'content type based deserialization' do
|
350
|
+
before do
|
351
|
+
Sneakers::ContentType.register(
|
352
|
+
content_type: 'application/json',
|
353
|
+
serializer: ->(_) {},
|
354
|
+
deserializer: ->(payload) { JSON.parse(payload) },
|
355
|
+
)
|
356
|
+
end
|
357
|
+
|
358
|
+
after do
|
359
|
+
Sneakers::ContentType.reset!
|
360
|
+
end
|
361
|
+
|
362
|
+
it 'should use the registered deserializer if the content type is in the metadata' do
|
363
|
+
w = DummyWorker.new(@queue, TestPool.new)
|
364
|
+
mock(w).work({'foo' => 'bar'}).once
|
365
|
+
w.do_work(nil, { content_type: 'application/json' }, '{"foo":"bar"}', nil)
|
366
|
+
end
|
367
|
+
|
368
|
+
it 'should use the registered deserializer if the content type is in the queue options' do
|
369
|
+
w = JSONWorker.new(@queue, TestPool.new)
|
370
|
+
mock(w).work({'foo' => 'bar'}).once
|
371
|
+
w.do_work(nil, {}, '{"foo":"bar"}', nil)
|
372
|
+
end
|
373
|
+
|
374
|
+
it 'should use the deserializer from the queue options even if the metadata has a different content type' do
|
375
|
+
w = JSONWorker.new(@queue, TestPool.new)
|
376
|
+
mock(w).work({'foo' => 'bar'}).once
|
377
|
+
w.do_work(nil, { content_type: 'not/real' }, '{"foo":"bar"}', nil)
|
378
|
+
end
|
379
|
+
end
|
380
|
+
|
333
381
|
it "should catch runtime exceptions from a bad work" do
|
334
382
|
w = AcksWorker.new(@queue, TestPool.new)
|
335
383
|
mock(w).work("msg").once{ raise "foo" }
|
@@ -432,6 +480,27 @@ describe Sneakers::Worker do
|
|
432
480
|
mock(@exchange).publish('msg', :routing_key => 'target', :expiration => 1).once
|
433
481
|
w.publish 'msg', :to_queue => 'target', :expiration => 1
|
434
482
|
end
|
483
|
+
|
484
|
+
describe 'content_type based serialization' do
|
485
|
+
before do
|
486
|
+
Sneakers::ContentType.register(
|
487
|
+
content_type: 'application/json',
|
488
|
+
serializer: ->(payload) { JSON.dump(payload) },
|
489
|
+
deserializer: ->(_) {},
|
490
|
+
)
|
491
|
+
end
|
492
|
+
|
493
|
+
after do
|
494
|
+
Sneakers::ContentType.reset!
|
495
|
+
end
|
496
|
+
|
497
|
+
it 'should be able to publish a message from working context' do
|
498
|
+
w = JSONPublishingWorker.new(@queue, TestPool.new)
|
499
|
+
mock(@exchange).publish('{"foo":"bar"}', :routing_key => 'target', :content_type => 'application/json').once
|
500
|
+
w.do_work(nil, {}, {'foo' => 'bar'}, nil)
|
501
|
+
end
|
502
|
+
end
|
503
|
+
|
435
504
|
end
|
436
505
|
|
437
506
|
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sneakers
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dotan Nahum
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-08-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: serverengine
|
@@ -30,28 +30,28 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: 2.
|
33
|
+
version: 2.7.0
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: 2.
|
40
|
+
version: 2.7.0
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: concurrent-ruby
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: '1.0'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: '1.0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: thor
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -267,6 +267,7 @@ files:
|
|
267
267
|
- lib/sneakers/concerns/logging.rb
|
268
268
|
- lib/sneakers/concerns/metrics.rb
|
269
269
|
- lib/sneakers/configuration.rb
|
270
|
+
- lib/sneakers/content_type.rb
|
270
271
|
- lib/sneakers/error_reporter.rb
|
271
272
|
- lib/sneakers/errors.rb
|
272
273
|
- lib/sneakers/handlers/maxretry.rb
|
@@ -294,6 +295,7 @@ files:
|
|
294
295
|
- spec/sneakers/concerns/logging_spec.rb
|
295
296
|
- spec/sneakers/concerns/metrics_spec.rb
|
296
297
|
- spec/sneakers/configuration_spec.rb
|
298
|
+
- spec/sneakers/content_type_spec.rb
|
297
299
|
- spec/sneakers/integration_spec.rb
|
298
300
|
- spec/sneakers/publisher_spec.rb
|
299
301
|
- spec/sneakers/queue_spec.rb
|
@@ -304,7 +306,8 @@ files:
|
|
304
306
|
- spec/sneakers/worker_spec.rb
|
305
307
|
- spec/spec_helper.rb
|
306
308
|
homepage: http://sneakers.io
|
307
|
-
licenses:
|
309
|
+
licenses:
|
310
|
+
- MIT
|
308
311
|
metadata: {}
|
309
312
|
post_install_message:
|
310
313
|
rdoc_options: []
|
@@ -322,7 +325,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
322
325
|
version: '0'
|
323
326
|
requirements: []
|
324
327
|
rubyforge_project:
|
325
|
-
rubygems_version: 2.
|
328
|
+
rubygems_version: 2.6.11
|
326
329
|
signing_key:
|
327
330
|
specification_version: 4
|
328
331
|
summary: Fast background processing framework for Ruby and RabbitMQ
|
@@ -333,6 +336,7 @@ test_files:
|
|
333
336
|
- spec/sneakers/concerns/logging_spec.rb
|
334
337
|
- spec/sneakers/concerns/metrics_spec.rb
|
335
338
|
- spec/sneakers/configuration_spec.rb
|
339
|
+
- spec/sneakers/content_type_spec.rb
|
336
340
|
- spec/sneakers/integration_spec.rb
|
337
341
|
- spec/sneakers/publisher_spec.rb
|
338
342
|
- spec/sneakers/queue_spec.rb
|