sneakers 2.5.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +4 -2
- data/examples/newrelic_metrics_worker.rb +2 -7
- data/lib/sneakers.rb +3 -2
- data/lib/sneakers/content_type.rb +47 -0
- data/lib/sneakers/handlers/maxretry.rb +1 -1
- data/lib/sneakers/publisher.rb +5 -3
- data/lib/sneakers/queue.rb +17 -4
- data/lib/sneakers/spawner.rb +1 -0
- data/lib/sneakers/version.rb +1 -1
- data/lib/sneakers/worker.rb +12 -8
- data/lib/sneakers/workergroup.rb +16 -2
- data/sneakers.gemspec +3 -2
- data/spec/sneakers/content_type_spec.rb +81 -0
- data/spec/sneakers/integration_spec.rb +42 -4
- data/spec/sneakers/publisher_spec.rb +22 -1
- data/spec/sneakers/worker_handlers_spec.rb +1 -6
- data/spec/sneakers/worker_spec.rb +75 -6
- data/spec/spec_helper.rb +1 -1
- metadata +13 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c52a398aecf272a36fcb3431f7b242a0d36dc0d1
|
4
|
+
data.tar.gz: c5974b0f7823f4d1e91c99d88adc3bbe50a2e572
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 85bc282b42260ebb6a08fdab01fc2396a30450f3cabb93bed27ccac801bc216e5e139613349cfb874a9c5c2a453234591759671de64c9f7654ddd5076384f746
|
7
|
+
data.tar.gz: 06fd1978121e6e59971152fd202e1cb2d5b13225d0f08f75583a5d021b0cfb8d52768102a007cef7d1dc191ed9264505a9e4157987054bc9631ee3eaaf749605
|
data/.travis.yml
CHANGED
@@ -25,8 +25,8 @@ class MetricsWorker
|
|
25
25
|
logger.info "FOUND <#{title}>"
|
26
26
|
ack!
|
27
27
|
end
|
28
|
-
|
29
|
-
add_transaction_tracer :work, name: 'MetricsWorker', params: 'args[0]'
|
28
|
+
|
29
|
+
add_transaction_tracer :work, name: 'MetricsWorker', params: 'args[0]', category: :task
|
30
30
|
|
31
31
|
private
|
32
32
|
|
@@ -36,10 +36,5 @@ class MetricsWorker
|
|
36
36
|
end
|
37
37
|
end
|
38
38
|
|
39
|
-
|
40
39
|
r = Sneakers::Runner.new([ MetricsWorker ])
|
41
40
|
r.run
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
data/lib/sneakers.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
|
-
require
|
2
|
-
require '
|
1
|
+
require 'sneakers/version'
|
2
|
+
require 'concurrent/executors'
|
3
3
|
require 'bunny'
|
4
4
|
require 'logger'
|
5
5
|
require 'serverengine'
|
@@ -17,6 +17,7 @@ require 'sneakers/support/production_formatter'
|
|
17
17
|
require 'sneakers/concerns/logging'
|
18
18
|
require 'sneakers/concerns/metrics'
|
19
19
|
require 'sneakers/handlers/oneshot'
|
20
|
+
require 'sneakers/content_type'
|
20
21
|
require 'sneakers/worker'
|
21
22
|
require 'sneakers/publisher'
|
22
23
|
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module Sneakers
|
2
|
+
class ContentType
|
3
|
+
def self.register(content_type: nil, serializer: nil, deserializer: nil)
|
4
|
+
# This can be removed when support is dropped for ruby 2.0 and replaced
|
5
|
+
# by a keyword arg with no default value
|
6
|
+
fail ArgumentError, 'missing keyword: content_type' if content_type.nil?
|
7
|
+
fail ArgumentError, 'missing keyword: serializer' if serializer.nil?
|
8
|
+
fail ArgumentError, 'missing keyword: deserializer' if deserializer.nil?
|
9
|
+
|
10
|
+
fail ArgumentError, "#{content_type} serializer must be a proc" unless serializer.is_a? Proc
|
11
|
+
fail ArgumentError, "#{content_type} deserializer must be a proc" unless deserializer.is_a? Proc
|
12
|
+
|
13
|
+
fail ArgumentError, "#{content_type} serializer must accept one argument, the payload" unless serializer.arity == 1
|
14
|
+
fail ArgumentError, "#{content_type} deserializer must accept one argument, the payload" unless deserializer.arity == 1
|
15
|
+
@_types[content_type] = new(serializer, deserializer)
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.serialize(payload, content_type)
|
19
|
+
return payload unless content_type
|
20
|
+
@_types[content_type].serializer.(payload)
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.deserialize(payload, content_type)
|
24
|
+
return payload unless content_type
|
25
|
+
@_types[content_type].deserializer.(payload)
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.reset!
|
29
|
+
@_types = Hash.new(
|
30
|
+
new(passthrough, passthrough)
|
31
|
+
)
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.passthrough
|
35
|
+
->(payload) { payload }
|
36
|
+
end
|
37
|
+
|
38
|
+
def initialize(serializer, deserializer)
|
39
|
+
@serializer = serializer
|
40
|
+
@deserializer = deserializer
|
41
|
+
end
|
42
|
+
|
43
|
+
attr_reader :serializer, :deserializer
|
44
|
+
|
45
|
+
reset!
|
46
|
+
end
|
47
|
+
end
|
@@ -135,7 +135,7 @@ module Sneakers
|
|
135
135
|
"#{log_prefix} msg=failing, retry_count=#{num_attempts}, reason=#{reason}"
|
136
136
|
end
|
137
137
|
data = {
|
138
|
-
error: reason,
|
138
|
+
error: reason.to_s,
|
139
139
|
num_attempts: num_attempts,
|
140
140
|
failed_at: Time.now.iso8601,
|
141
141
|
payload: Base64.encode64(msg.to_s),
|
data/lib/sneakers/publisher.rb
CHANGED
@@ -12,7 +12,7 @@ module Sneakers
|
|
12
12
|
to_queue = options.delete(:to_queue)
|
13
13
|
options[:routing_key] ||= to_queue
|
14
14
|
Sneakers.logger.info {"publishing <#{msg}> to [#{options[:routing_key]}]"}
|
15
|
-
@exchange.publish(msg, options)
|
15
|
+
@exchange.publish(ContentType.serialize(msg, options[:content_type]), options)
|
16
16
|
end
|
17
17
|
|
18
18
|
|
@@ -34,8 +34,10 @@ module Sneakers
|
|
34
34
|
end
|
35
35
|
|
36
36
|
def create_bunny_connection
|
37
|
-
Bunny.new(@opts[:amqp], :vhost => @opts[:vhost],
|
37
|
+
Bunny.new(@opts[:amqp], :vhost => @opts[:vhost],
|
38
|
+
:heartbeat => @opts[:heartbeat],
|
39
|
+
:properties => @opts.fetch(:properties, {}),
|
40
|
+
:logger => Sneakers::logger)
|
38
41
|
end
|
39
42
|
end
|
40
43
|
end
|
41
|
-
|
data/lib/sneakers/queue.rb
CHANGED
@@ -56,13 +56,26 @@ class Sneakers::Queue
|
|
56
56
|
end
|
57
57
|
|
58
58
|
def unsubscribe
|
59
|
-
|
60
|
-
|
61
|
-
|
59
|
+
return unless @consumer
|
60
|
+
|
61
|
+
# TODO: should we simply close the channel here?
|
62
|
+
Sneakers.logger.info("Queue: will try to cancel consumer #{@consumer.inspect}")
|
63
|
+
cancel_ok = @consumer.cancel
|
64
|
+
if cancel_ok
|
65
|
+
Sneakers.logger.info "Queue: consumer #{cancel_ok.consumer_tag} cancelled"
|
66
|
+
@consumer = nil
|
67
|
+
else
|
68
|
+
Sneakers.logger.warn "Queue: could not cancel consumer #{@consumer.inspect}"
|
69
|
+
sleep(1)
|
70
|
+
unsubscribe
|
71
|
+
end
|
62
72
|
end
|
63
73
|
|
64
74
|
def create_bunny_connection
|
65
|
-
Bunny.new(@opts[:amqp], :vhost => @opts[:vhost],
|
75
|
+
Bunny.new(@opts[:amqp], :vhost => @opts[:vhost],
|
76
|
+
:heartbeat => @opts[:heartbeat],
|
77
|
+
:properties => @opts.fetch(:properties, {}),
|
78
|
+
:logger => Sneakers::logger)
|
66
79
|
end
|
67
80
|
private :create_bunny_connection
|
68
81
|
end
|
data/lib/sneakers/spawner.rb
CHANGED
@@ -8,6 +8,7 @@ module Sneakers
|
|
8
8
|
unless File.exists?(worker_group_config_file)
|
9
9
|
puts "No worker group file found."
|
10
10
|
puts "Specify via ENV 'WORKER_GROUP_CONFIG' or by convention ./config/sneaker_worker_groups.yml"
|
11
|
+
Kernel.exit(1)
|
11
12
|
end
|
12
13
|
@pids = []
|
13
14
|
@exec_string = "bundle exec rake sneakers:run"
|
data/lib/sneakers/version.rb
CHANGED
data/lib/sneakers/worker.rb
CHANGED
@@ -19,8 +19,9 @@ module Sneakers
|
|
19
19
|
|
20
20
|
@should_ack = opts[:ack]
|
21
21
|
@timeout_after = opts[:timeout_job_after]
|
22
|
-
@pool = pool ||
|
22
|
+
@pool = pool || Concurrent::FixedThreadPool.new(opts[:threads])
|
23
23
|
@call_with_params = respond_to?(:work_with_params)
|
24
|
+
@content_type = opts[:content_type]
|
24
25
|
|
25
26
|
@queue = queue || Sneakers::Queue.new(
|
26
27
|
queue_name,
|
@@ -39,13 +40,13 @@ module Sneakers
|
|
39
40
|
to_queue = opts.delete(:to_queue)
|
40
41
|
opts[:routing_key] ||= to_queue
|
41
42
|
return unless opts[:routing_key]
|
42
|
-
@queue.exchange.publish(msg, opts)
|
43
|
+
@queue.exchange.publish(Sneakers::ContentType.serialize(msg, opts[:content_type]), opts)
|
43
44
|
end
|
44
45
|
|
45
46
|
def do_work(delivery_info, metadata, msg, handler)
|
46
47
|
worker_trace "Working off: #{msg.inspect}"
|
47
48
|
|
48
|
-
@pool.
|
49
|
+
@pool.post do
|
49
50
|
res = nil
|
50
51
|
error = nil
|
51
52
|
|
@@ -53,10 +54,11 @@ module Sneakers
|
|
53
54
|
metrics.increment("work.#{self.class.name}.started")
|
54
55
|
Timeout.timeout(@timeout_after, WorkerTimeout) do
|
55
56
|
metrics.timing("work.#{self.class.name}.time") do
|
57
|
+
deserialized_msg = ContentType.deserialize(msg, @content_type || metadata && metadata[:content_type])
|
56
58
|
if @call_with_params
|
57
|
-
res = work_with_params(
|
59
|
+
res = work_with_params(deserialized_msg, delivery_info, metadata)
|
58
60
|
else
|
59
|
-
res = work(
|
61
|
+
res = work(deserialized_msg)
|
60
62
|
end
|
61
63
|
end
|
62
64
|
end
|
@@ -91,14 +93,15 @@ module Sneakers
|
|
91
93
|
end
|
92
94
|
|
93
95
|
metrics.increment("work.#{self.class.name}.ended")
|
94
|
-
end #
|
96
|
+
end #post
|
95
97
|
end
|
96
98
|
|
97
99
|
def stop
|
98
|
-
worker_trace "Stopping worker: shutting down thread pool."
|
99
|
-
@pool.shutdown
|
100
100
|
worker_trace "Stopping worker: unsubscribing."
|
101
101
|
@queue.unsubscribe
|
102
|
+
worker_trace "Stopping worker: shutting down thread pool."
|
103
|
+
@pool.shutdown
|
104
|
+
@pool.wait_for_termination
|
102
105
|
worker_trace "Stopping worker: I'm gone."
|
103
106
|
end
|
104
107
|
|
@@ -135,6 +138,7 @@ module Sneakers
|
|
135
138
|
|
136
139
|
def enqueue(msg, opts={})
|
137
140
|
opts[:routing_key] ||= @queue_opts[:routing_key]
|
141
|
+
opts[:content_type] ||= @queue_opts[:content_type]
|
138
142
|
opts[:to_queue] ||= @queue_name
|
139
143
|
|
140
144
|
publisher.publish(msg, opts)
|
data/lib/sneakers/workergroup.rb
CHANGED
@@ -21,7 +21,7 @@ module Sneakers
|
|
21
21
|
|
22
22
|
# Allocate single thread pool if share_threads is set. This improves load balancing
|
23
23
|
# when used with many workers.
|
24
|
-
pool = config[:share_threads] ?
|
24
|
+
pool = config[:share_threads] ? Concurrent::FixedThreadPool.new(config[:threads]) : nil
|
25
25
|
|
26
26
|
worker_classes = config[:worker_classes]
|
27
27
|
|
@@ -29,7 +29,13 @@ module Sneakers
|
|
29
29
|
worker_classes = worker_classes.call
|
30
30
|
end
|
31
31
|
|
32
|
-
|
32
|
+
# if we don't provide a connection to a worker,
|
33
|
+
# the queue used in the worker will create a new one
|
34
|
+
# so if we want to have a shared bunny connection for the workers
|
35
|
+
# we must create it here
|
36
|
+
bunny_connection = create_connection_or_nil
|
37
|
+
|
38
|
+
@workers = worker_classes.map{|w| w.new(nil, pool, {connection: bunny_connection}) }
|
33
39
|
# if more than one worker this should be per worker
|
34
40
|
# accumulate clients and consumers as well
|
35
41
|
@workers.each do |worker|
|
@@ -52,5 +58,13 @@ module Sneakers
|
|
52
58
|
@stop_flag.set!
|
53
59
|
end
|
54
60
|
|
61
|
+
def create_bunny_connection
|
62
|
+
Bunny.new(Sneakers::CONFIG[:amqp], :vhost => Sneakers::CONFIG[:vhost], :heartbeat => Sneakers::CONFIG[:heartbeat], :logger => Sneakers::logger)
|
63
|
+
end
|
64
|
+
|
65
|
+
def create_connection_or_nil
|
66
|
+
config[:share_bunny_connection] ? create_bunny_connection : nil
|
67
|
+
end
|
68
|
+
private :create_bunny_connection, :create_connection_or_nil
|
55
69
|
end
|
56
70
|
end
|
data/sneakers.gemspec
CHANGED
@@ -11,6 +11,7 @@ Gem::Specification.new do |gem|
|
|
11
11
|
gem.description = %q( Fast background processing framework for Ruby and RabbitMQ )
|
12
12
|
gem.summary = %q( Fast background processing framework for Ruby and RabbitMQ )
|
13
13
|
gem.homepage = 'http://sneakers.io'
|
14
|
+
gem.license = 'MIT'
|
14
15
|
gem.required_ruby_version = Gem::Requirement.new(">= 2.0")
|
15
16
|
|
16
17
|
gem.files = `git ls-files`.split($/).reject { |f| f == 'Gemfile.lock' }
|
@@ -18,8 +19,8 @@ Gem::Specification.new do |gem|
|
|
18
19
|
gem.test_files = gem.files.grep(/^(test|spec|features)\//)
|
19
20
|
gem.require_paths = ['lib']
|
20
21
|
gem.add_dependency 'serverengine', '~> 1.5.11'
|
21
|
-
gem.add_dependency 'bunny', '~> 2.
|
22
|
-
gem.add_dependency '
|
22
|
+
gem.add_dependency 'bunny', '~> 2.7.0'
|
23
|
+
gem.add_dependency 'concurrent-ruby', '~> 1.0'
|
23
24
|
gem.add_dependency 'thor'
|
24
25
|
|
25
26
|
# for integration environment (see .travis.yml and integration_spec)
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'sneakers/content_type'
|
3
|
+
require 'base64'
|
4
|
+
|
5
|
+
describe Sneakers::ContentType do
|
6
|
+
after do
|
7
|
+
Sneakers::ContentType.reset!
|
8
|
+
end
|
9
|
+
|
10
|
+
describe '.deserialize' do
|
11
|
+
it 'uses the given deserializer' do
|
12
|
+
Sneakers::ContentType.register(
|
13
|
+
content_type: 'application/json',
|
14
|
+
serializer: ->(_) {},
|
15
|
+
deserializer: ->(payload) { JSON.parse(payload) },
|
16
|
+
)
|
17
|
+
|
18
|
+
Sneakers::ContentType.deserialize('{"foo":"bar"}', 'application/json').must_equal('foo' => 'bar')
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe '.serialize' do
|
23
|
+
it 'uses the given serializer' do
|
24
|
+
Sneakers::ContentType.register(
|
25
|
+
content_type: 'application/json',
|
26
|
+
serializer: ->(payload) { JSON.dump(payload) },
|
27
|
+
deserializer: ->(_) {},
|
28
|
+
)
|
29
|
+
|
30
|
+
Sneakers::ContentType.serialize({ 'foo' => 'bar' }, 'application/json').must_equal('{"foo":"bar"}')
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'passes the payload through by default' do
|
34
|
+
payload = "just some text"
|
35
|
+
Sneakers::ContentType.serialize(payload, 'unknown/type').must_equal(payload)
|
36
|
+
Sneakers::ContentType.deserialize(payload, 'unknown/type').must_equal(payload)
|
37
|
+
Sneakers::ContentType.serialize(payload, nil).must_equal(payload)
|
38
|
+
Sneakers::ContentType.deserialize(payload, nil).must_equal(payload)
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'passes the payload through if type not found' do
|
42
|
+
Sneakers::ContentType.register(content_type: 'found/type', serializer: ->(_) {}, deserializer: ->(_) {})
|
43
|
+
payload = "just some text"
|
44
|
+
|
45
|
+
Sneakers::ContentType.serialize(payload, 'unknown/type').must_equal(payload)
|
46
|
+
Sneakers::ContentType.deserialize(payload, 'unknown/type').must_equal(payload)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
describe '.register' do
|
51
|
+
it 'provides a mechnism to register a given type' do
|
52
|
+
Sneakers::ContentType.register(
|
53
|
+
content_type: 'text/base64',
|
54
|
+
serializer: ->(payload) { Base64.encode64(payload) },
|
55
|
+
deserializer: ->(payload) { Base64.decode64(payload) },
|
56
|
+
)
|
57
|
+
|
58
|
+
ct = Sneakers::ContentType
|
59
|
+
ct.deserialize(ct.serialize('hello world', 'text/base64'), 'text/base64').must_equal('hello world')
|
60
|
+
end
|
61
|
+
|
62
|
+
it 'requires a content type' do
|
63
|
+
proc { Sneakers::ContentType.register(serializer: -> { }, deserializer: -> { }) }.must_raise ArgumentError
|
64
|
+
end
|
65
|
+
|
66
|
+
it 'expects serializer and deserializer to be present' do
|
67
|
+
proc { Sneakers::ContentType.register(content_type: 'foo', deserializer: -> { }) }.must_raise ArgumentError
|
68
|
+
proc { Sneakers::ContentType.register(content_type: 'foo', serializer: -> { }) }.must_raise ArgumentError
|
69
|
+
end
|
70
|
+
|
71
|
+
it 'expects serializer and deserializer to be a proc' do
|
72
|
+
proc { Sneakers::ContentType.register(content_type: 'foo', serializer: 'not a proc', deserializer: ->(_) { }) }.must_raise ArgumentError
|
73
|
+
proc { Sneakers::ContentType.register(content_type: 'foo', serializer: ->(_) {}, deserializer: 'not a proc' ) }.must_raise ArgumentError
|
74
|
+
end
|
75
|
+
|
76
|
+
it 'expects serializer and deserializer to have the correct arity' do
|
77
|
+
proc { Sneakers::ContentType.register(content_type: 'foo', serializer: ->(_,_) {}, deserializer: ->(_) {}) }.must_raise ArgumentError
|
78
|
+
proc { Sneakers::ContentType.register(content_type: 'foo', serializer: ->(_) {}, deserializer: ->() {} ) }.must_raise ArgumentError
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
@@ -65,7 +65,7 @@ describe "integration" do
|
|
65
65
|
return
|
66
66
|
end
|
67
67
|
end
|
68
|
-
|
68
|
+
|
69
69
|
integration_log "failed test. killing off workers."
|
70
70
|
Process.kill("TERM", pid)
|
71
71
|
sleep 1
|
@@ -85,6 +85,44 @@ describe "integration" do
|
|
85
85
|
pid
|
86
86
|
end
|
87
87
|
|
88
|
+
def any_consumers
|
89
|
+
rmq_addr = compose_or_localhost("rabbitmq")
|
90
|
+
result = false
|
91
|
+
begin
|
92
|
+
admin = RabbitMQ::HTTP::Client.new("http://#{rmq_addr}:15672/", username: "guest", password: "guest")
|
93
|
+
qs = admin.list_queues
|
94
|
+
qs.each do |q|
|
95
|
+
if q.name.start_with? 'integration_'
|
96
|
+
puts "We see #{q.consumers} consumers on #{q.name}"
|
97
|
+
return true if q.consumers > 0
|
98
|
+
end
|
99
|
+
end
|
100
|
+
return false
|
101
|
+
rescue
|
102
|
+
puts "Rabbitmq admin seems to not exist? you better be running this on Travis or Docker. proceeding.\n#{$!}"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
it 'should be possible to terminate when queue is full' do
|
107
|
+
job_count = 40000
|
108
|
+
|
109
|
+
pid = start_worker(IntegrationWorker)
|
110
|
+
Process.kill("TERM", pid)
|
111
|
+
|
112
|
+
integration_log "publishing #{job_count} messages..."
|
113
|
+
p = Sneakers::Publisher.new
|
114
|
+
job_count.times do |i|
|
115
|
+
p.publish("m #{i}", to_queue: IntegrationWorker.queue_name)
|
116
|
+
end
|
117
|
+
|
118
|
+
pid = start_worker(IntegrationWorker)
|
119
|
+
any_consumers.must_equal true
|
120
|
+
integration_log "Killing #{pid} now!"
|
121
|
+
Process.kill("TERM", pid)
|
122
|
+
sleep(2)
|
123
|
+
any_consumers.must_equal false
|
124
|
+
end
|
125
|
+
|
88
126
|
it 'should pull down 100 jobs from a real queue' do
|
89
127
|
job_count = 100
|
90
128
|
|
@@ -95,7 +133,7 @@ describe "integration" do
|
|
95
133
|
job_count.times do |i|
|
96
134
|
p.publish("m #{i}", to_queue: IntegrationWorker.queue_name)
|
97
135
|
end
|
98
|
-
|
136
|
+
|
99
137
|
assert_all_accounted_for(
|
100
138
|
queue: IntegrationWorker.queue_name,
|
101
139
|
pid: pid,
|
@@ -103,8 +141,8 @@ describe "integration" do
|
|
103
141
|
jobs: job_count,
|
104
142
|
)
|
105
143
|
end
|
106
|
-
|
144
|
+
|
107
145
|
end
|
108
146
|
end
|
109
|
-
|
147
|
+
|
110
148
|
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
require 'sneakers'
|
3
|
+
require 'serverengine'
|
3
4
|
|
4
5
|
describe Sneakers::Publisher do
|
5
6
|
let :pub_vars do
|
@@ -75,6 +76,7 @@ describe Sneakers::Publisher do
|
|
75
76
|
exchange: 'another_exchange',
|
76
77
|
exchange_options: { :type => :topic, :arguments => { 'x-arg' => 'value' } },
|
77
78
|
log: logger,
|
79
|
+
properties: { key: "value" },
|
78
80
|
durable: false)
|
79
81
|
|
80
82
|
channel = Object.new
|
@@ -86,7 +88,7 @@ describe Sneakers::Publisher do
|
|
86
88
|
mock(bunny).start
|
87
89
|
mock(bunny).create_channel { channel }
|
88
90
|
|
89
|
-
mock(Bunny).new('amqp://someuser:somepassword@somehost:5672', heartbeat: 1, vhost: '/', logger: logger) { bunny }
|
91
|
+
mock(Bunny).new('amqp://someuser:somepassword@somehost:5672', heartbeat: 1, vhost: '/', logger: logger, properties: { key: "value" }) { bunny }
|
90
92
|
|
91
93
|
p = Sneakers::Publisher.new
|
92
94
|
|
@@ -122,5 +124,24 @@ describe Sneakers::Publisher do
|
|
122
124
|
p.publish('test msg', my_vars)
|
123
125
|
p.instance_variable_get(:@bunny).must_equal existing_session
|
124
126
|
end
|
127
|
+
|
128
|
+
it 'should publish using the content type serializer' do
|
129
|
+
Sneakers::ContentType.register(
|
130
|
+
content_type: 'application/json',
|
131
|
+
serializer: ->(payload) { JSON.dump(payload) },
|
132
|
+
deserializer: ->(_) {},
|
133
|
+
)
|
134
|
+
|
135
|
+
xchg = Object.new
|
136
|
+
mock(xchg).publish('{"foo":"bar"}', routing_key: 'downloads', content_type: 'application/json')
|
137
|
+
|
138
|
+
p = Sneakers::Publisher.new
|
139
|
+
p.instance_variable_set(:@exchange, xchg)
|
140
|
+
|
141
|
+
mock(p).ensure_connection! {}
|
142
|
+
p.publish({ 'foo' => 'bar' }, to_queue: 'downloads', content_type: 'application/json')
|
143
|
+
|
144
|
+
Sneakers::ContentType.reset!
|
145
|
+
end
|
125
146
|
end
|
126
147
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
require 'sneakers'
|
3
3
|
require 'timeout'
|
4
|
+
require 'serverengine'
|
4
5
|
|
5
6
|
class DummyWorker
|
6
7
|
include Sneakers::Worker
|
@@ -75,7 +76,16 @@ class PublishingWorker
|
|
75
76
|
end
|
76
77
|
end
|
77
78
|
|
79
|
+
class JSONPublishingWorker
|
80
|
+
include Sneakers::Worker
|
81
|
+
from_queue 'defaults',
|
82
|
+
:ack => false,
|
83
|
+
:exchange => 'foochange'
|
78
84
|
|
85
|
+
def work(msg)
|
86
|
+
publish msg, :to_queue => 'target', :content_type => 'application/json'
|
87
|
+
end
|
88
|
+
end
|
79
89
|
|
80
90
|
class LoggingWorker
|
81
91
|
include Sneakers::Worker
|
@@ -87,6 +97,15 @@ class LoggingWorker
|
|
87
97
|
end
|
88
98
|
end
|
89
99
|
|
100
|
+
class JSONWorker
|
101
|
+
include Sneakers::Worker
|
102
|
+
from_queue 'defaults',
|
103
|
+
:ack => false,
|
104
|
+
:content_type => 'application/json'
|
105
|
+
|
106
|
+
def work(msg)
|
107
|
+
end
|
108
|
+
end
|
90
109
|
|
91
110
|
class MetricsWorker
|
92
111
|
include Sneakers::Worker
|
@@ -123,11 +142,7 @@ class WithDeprecatedExchangeOptionsWorker
|
|
123
142
|
end
|
124
143
|
end
|
125
144
|
|
126
|
-
|
127
|
-
def process(*args,&block)
|
128
|
-
block.call
|
129
|
-
end
|
130
|
-
end
|
145
|
+
TestPool ||= Concurrent::ImmediateExecutor
|
131
146
|
|
132
147
|
def with_test_queuefactory(ctx, ack=true, msg=nil, nowork=false)
|
133
148
|
qf = Object.new
|
@@ -164,7 +179,8 @@ describe Sneakers::Worker do
|
|
164
179
|
mock(Sneakers::Publisher).new(DummyWorker.queue_opts) do
|
165
180
|
mock(Object.new).publish(message, {
|
166
181
|
:routing_key => 'test.routing.key',
|
167
|
-
:to_queue => 'downloads'
|
182
|
+
:to_queue => 'downloads',
|
183
|
+
:content_type => nil,
|
168
184
|
})
|
169
185
|
end
|
170
186
|
|
@@ -330,6 +346,38 @@ describe Sneakers::Worker do
|
|
330
346
|
w.do_work(nil, nil, "msg", nil)
|
331
347
|
end
|
332
348
|
|
349
|
+
describe 'content type based deserialization' do
|
350
|
+
before do
|
351
|
+
Sneakers::ContentType.register(
|
352
|
+
content_type: 'application/json',
|
353
|
+
serializer: ->(_) {},
|
354
|
+
deserializer: ->(payload) { JSON.parse(payload) },
|
355
|
+
)
|
356
|
+
end
|
357
|
+
|
358
|
+
after do
|
359
|
+
Sneakers::ContentType.reset!
|
360
|
+
end
|
361
|
+
|
362
|
+
it 'should use the registered deserializer if the content type is in the metadata' do
|
363
|
+
w = DummyWorker.new(@queue, TestPool.new)
|
364
|
+
mock(w).work({'foo' => 'bar'}).once
|
365
|
+
w.do_work(nil, { content_type: 'application/json' }, '{"foo":"bar"}', nil)
|
366
|
+
end
|
367
|
+
|
368
|
+
it 'should use the registered deserializer if the content type is in the queue options' do
|
369
|
+
w = JSONWorker.new(@queue, TestPool.new)
|
370
|
+
mock(w).work({'foo' => 'bar'}).once
|
371
|
+
w.do_work(nil, {}, '{"foo":"bar"}', nil)
|
372
|
+
end
|
373
|
+
|
374
|
+
it 'should use the deserializer from the queue options even if the metadata has a different content type' do
|
375
|
+
w = JSONWorker.new(@queue, TestPool.new)
|
376
|
+
mock(w).work({'foo' => 'bar'}).once
|
377
|
+
w.do_work(nil, { content_type: 'not/real' }, '{"foo":"bar"}', nil)
|
378
|
+
end
|
379
|
+
end
|
380
|
+
|
333
381
|
it "should catch runtime exceptions from a bad work" do
|
334
382
|
w = AcksWorker.new(@queue, TestPool.new)
|
335
383
|
mock(w).work("msg").once{ raise "foo" }
|
@@ -432,6 +480,27 @@ describe Sneakers::Worker do
|
|
432
480
|
mock(@exchange).publish('msg', :routing_key => 'target', :expiration => 1).once
|
433
481
|
w.publish 'msg', :to_queue => 'target', :expiration => 1
|
434
482
|
end
|
483
|
+
|
484
|
+
describe 'content_type based serialization' do
|
485
|
+
before do
|
486
|
+
Sneakers::ContentType.register(
|
487
|
+
content_type: 'application/json',
|
488
|
+
serializer: ->(payload) { JSON.dump(payload) },
|
489
|
+
deserializer: ->(_) {},
|
490
|
+
)
|
491
|
+
end
|
492
|
+
|
493
|
+
after do
|
494
|
+
Sneakers::ContentType.reset!
|
495
|
+
end
|
496
|
+
|
497
|
+
it 'should be able to publish a message from working context' do
|
498
|
+
w = JSONPublishingWorker.new(@queue, TestPool.new)
|
499
|
+
mock(@exchange).publish('{"foo":"bar"}', :routing_key => 'target', :content_type => 'application/json').once
|
500
|
+
w.do_work(nil, {}, {'foo' => 'bar'}, nil)
|
501
|
+
end
|
502
|
+
end
|
503
|
+
|
435
504
|
end
|
436
505
|
|
437
506
|
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sneakers
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dotan Nahum
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-08-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: serverengine
|
@@ -30,28 +30,28 @@ dependencies:
|
|
30
30
|
requirements:
|
31
31
|
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: 2.
|
33
|
+
version: 2.7.0
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
38
|
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: 2.
|
40
|
+
version: 2.7.0
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
42
|
+
name: concurrent-ruby
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
47
|
+
version: '1.0'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
54
|
+
version: '1.0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: thor
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -267,6 +267,7 @@ files:
|
|
267
267
|
- lib/sneakers/concerns/logging.rb
|
268
268
|
- lib/sneakers/concerns/metrics.rb
|
269
269
|
- lib/sneakers/configuration.rb
|
270
|
+
- lib/sneakers/content_type.rb
|
270
271
|
- lib/sneakers/error_reporter.rb
|
271
272
|
- lib/sneakers/errors.rb
|
272
273
|
- lib/sneakers/handlers/maxretry.rb
|
@@ -294,6 +295,7 @@ files:
|
|
294
295
|
- spec/sneakers/concerns/logging_spec.rb
|
295
296
|
- spec/sneakers/concerns/metrics_spec.rb
|
296
297
|
- spec/sneakers/configuration_spec.rb
|
298
|
+
- spec/sneakers/content_type_spec.rb
|
297
299
|
- spec/sneakers/integration_spec.rb
|
298
300
|
- spec/sneakers/publisher_spec.rb
|
299
301
|
- spec/sneakers/queue_spec.rb
|
@@ -304,7 +306,8 @@ files:
|
|
304
306
|
- spec/sneakers/worker_spec.rb
|
305
307
|
- spec/spec_helper.rb
|
306
308
|
homepage: http://sneakers.io
|
307
|
-
licenses:
|
309
|
+
licenses:
|
310
|
+
- MIT
|
308
311
|
metadata: {}
|
309
312
|
post_install_message:
|
310
313
|
rdoc_options: []
|
@@ -322,7 +325,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
322
325
|
version: '0'
|
323
326
|
requirements: []
|
324
327
|
rubyforge_project:
|
325
|
-
rubygems_version: 2.
|
328
|
+
rubygems_version: 2.6.11
|
326
329
|
signing_key:
|
327
330
|
specification_version: 4
|
328
331
|
summary: Fast background processing framework for Ruby and RabbitMQ
|
@@ -333,6 +336,7 @@ test_files:
|
|
333
336
|
- spec/sneakers/concerns/logging_spec.rb
|
334
337
|
- spec/sneakers/concerns/metrics_spec.rb
|
335
338
|
- spec/sneakers/configuration_spec.rb
|
339
|
+
- spec/sneakers/content_type_spec.rb
|
336
340
|
- spec/sneakers/integration_spec.rb
|
337
341
|
- spec/sneakers/publisher_spec.rb
|
338
342
|
- spec/sneakers/queue_spec.rb
|