shoryuken 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +53 -5
- data/lib/shoryuken.rb +26 -25
- data/lib/shoryuken/cli.rb +22 -17
- data/lib/shoryuken/default_worker_registry.rb +46 -0
- data/lib/shoryuken/extensions/active_job_adapter.rb +63 -0
- data/lib/shoryuken/fetcher.rb +2 -3
- data/lib/shoryuken/launcher.rb +4 -2
- data/lib/shoryuken/manager.rb +26 -13
- data/lib/shoryuken/middleware/chain.rb +4 -0
- data/lib/shoryuken/middleware/server/timing.rb +1 -1
- data/lib/shoryuken/processor.rb +33 -7
- data/lib/shoryuken/util.rb +13 -0
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker.rb +19 -1
- data/lib/shoryuken/worker_registry.rb +34 -0
- data/shoryuken.gemspec +2 -2
- data/spec/shoryuken/default_worker_registry_spec.rb +83 -0
- data/spec/shoryuken/manager_spec.rb +6 -0
- data/spec/shoryuken/processor_spec.rb +65 -8
- data/spec/shoryuken/util_spec.rb +10 -0
- data/spec/shoryuken/worker_spec.rb +100 -2
- data/spec/shoryuken_spec.rb +9 -9
- data/spec/spec_helper.rb +1 -1
- metadata +10 -8
- data/lib/shoryuken/worker_loader.rb +0 -17
- data/spec/shoryuken/worker_loader_spec.rb +0 -27
@@ -5,7 +5,7 @@ module Shoryuken
|
|
5
5
|
include Util
|
6
6
|
|
7
7
|
def call(worker, queue, sqs_msg, body)
|
8
|
-
Shoryuken::Logging.with_context("#{worker.class
|
8
|
+
Shoryuken::Logging.with_context("#{worker_name(worker.class, sqs_msg, body)}/#{queue}/#{sqs_msg.id}") do
|
9
9
|
begin
|
10
10
|
started_at = Time.now
|
11
11
|
|
data/lib/shoryuken/processor.rb
CHANGED
@@ -9,22 +9,48 @@ module Shoryuken
|
|
9
9
|
@manager = manager
|
10
10
|
end
|
11
11
|
|
12
|
+
attr_accessor :proxy_id
|
13
|
+
|
12
14
|
def process(queue, sqs_msg)
|
13
|
-
|
15
|
+
@manager.async.real_thread(proxy_id, Thread.current)
|
16
|
+
|
17
|
+
worker = Shoryuken.worker_registry.fetch_worker(queue, sqs_msg)
|
18
|
+
|
19
|
+
timer = auto_visibility_timeout(queue, sqs_msg, worker.class)
|
14
20
|
|
15
|
-
|
16
|
-
|
21
|
+
begin
|
22
|
+
defer do
|
23
|
+
body = get_body(worker.class, sqs_msg)
|
17
24
|
|
18
|
-
|
19
|
-
|
25
|
+
worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
|
26
|
+
worker.perform(sqs_msg, body)
|
27
|
+
end
|
20
28
|
end
|
21
|
-
end
|
22
29
|
|
23
|
-
|
30
|
+
@manager.async.processor_done(queue, current_actor)
|
31
|
+
ensure
|
32
|
+
timer.cancel if timer
|
33
|
+
end
|
24
34
|
end
|
25
35
|
|
26
36
|
private
|
27
37
|
|
38
|
+
def auto_visibility_timeout(queue, sqs_msg, worker_class)
|
39
|
+
if worker_class.auto_visibility_timeout?
|
40
|
+
timer = every(worker_class.visibility_timeout_heartbeat) do
|
41
|
+
begin
|
42
|
+
logger.debug "Extending message #{worker_name(worker_class, sqs_msg)}/#{queue}/#{sqs_msg.id} visibility timeout to #{worker_class.extended_visibility_timeout}"
|
43
|
+
|
44
|
+
sqs_msg.visibility_timeout = worker_class.extended_visibility_timeout
|
45
|
+
rescue => e
|
46
|
+
logger.error "Could not auto extend the message #{worker_class}/#{queue}/#{sqs_msg.id} visibility timeout. Error: #{e.message}"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
timer
|
52
|
+
end
|
53
|
+
|
28
54
|
def get_body(worker_class, sqs_msg)
|
29
55
|
if sqs_msg.is_a? Array
|
30
56
|
sqs_msg.map { |m| parse_body(worker_class, m) }
|
data/lib/shoryuken/util.rb
CHANGED
@@ -23,5 +23,18 @@ module Shoryuken
|
|
23
23
|
queue_and_weights
|
24
24
|
end.to_a
|
25
25
|
end
|
26
|
+
|
27
|
+
def worker_name(worker_class, sqs_msg, body = nil)
|
28
|
+
if defined?(::ActiveJob) \
|
29
|
+
&& !sqs_msg.is_a?(Array) \
|
30
|
+
&& sqs_msg.message_attributes \
|
31
|
+
&& sqs_msg.message_attributes['shoryuken_class'] \
|
32
|
+
&& sqs_msg.message_attributes['shoryuken_class'][:string_value] == ActiveJob::QueueAdapters::ShoryukenAdapter::JobWrapper.to_s
|
33
|
+
|
34
|
+
"ActiveJob/#{body['job_class'].constantize}"
|
35
|
+
else
|
36
|
+
worker_class.to_s
|
37
|
+
end
|
38
|
+
end
|
26
39
|
end
|
27
40
|
end
|
data/lib/shoryuken/version.rb
CHANGED
data/lib/shoryuken/worker.rb
CHANGED
@@ -30,6 +30,12 @@ module Shoryuken
|
|
30
30
|
|
31
31
|
alias_method :perform_at, :perform_in
|
32
32
|
|
33
|
+
def server_middleware
|
34
|
+
@server_chain ||= Shoryuken.server_middleware.dup
|
35
|
+
yield @server_chain if block_given?
|
36
|
+
@server_chain
|
37
|
+
end
|
38
|
+
|
33
39
|
def shoryuken_options(opts = {})
|
34
40
|
@shoryuken_options = get_shoryuken_options.merge(stringify_keys(Hash(opts)))
|
35
41
|
queue = @shoryuken_options['queue']
|
@@ -41,8 +47,20 @@ module Shoryuken
|
|
41
47
|
Shoryuken.register_worker(queue, self)
|
42
48
|
end
|
43
49
|
|
50
|
+
def auto_visibility_timeout?
|
51
|
+
!!get_shoryuken_options['auto_visibility_timeout']
|
52
|
+
end
|
53
|
+
|
54
|
+
def visibility_timeout_heartbeat
|
55
|
+
extended_visibility_timeout - 5
|
56
|
+
end
|
57
|
+
|
58
|
+
def extended_visibility_timeout
|
59
|
+
Shoryuken::Client.visibility_timeout(get_shoryuken_options['queue'])
|
60
|
+
end
|
61
|
+
|
44
62
|
def get_shoryuken_options # :nodoc:
|
45
|
-
@shoryuken_options ||
|
63
|
+
@shoryuken_options || Shoryuken.default_worker_options
|
46
64
|
end
|
47
65
|
|
48
66
|
def stringify_keys(hash) # :nodoc:
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
class WorkerRegistry
|
3
|
+
def batch_receive_messages?(queue)
|
4
|
+
# true if the workers for queue support batch processing of messages
|
5
|
+
fail NotImplementedError
|
6
|
+
end
|
7
|
+
|
8
|
+
def clear
|
9
|
+
# must remove all worker registrations
|
10
|
+
fail NotImplementedError
|
11
|
+
end
|
12
|
+
|
13
|
+
def fetch_worker(queue, message)
|
14
|
+
# must return an instance of the worker that handles
|
15
|
+
# message received on queue
|
16
|
+
fail NotImplementedError
|
17
|
+
end
|
18
|
+
|
19
|
+
def queues
|
20
|
+
# must return a list of all queues with registered workers
|
21
|
+
fail NotImplementedError
|
22
|
+
end
|
23
|
+
|
24
|
+
def register_worker(queue, clazz)
|
25
|
+
# must register the worker as a consumer of messages from queue
|
26
|
+
fail NotImplementedError
|
27
|
+
end
|
28
|
+
|
29
|
+
def workers(queue)
|
30
|
+
# must return the list of workers registered for queue, or []
|
31
|
+
fail NotImplementedError
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
data/shoryuken.gemspec
CHANGED
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
|
|
10
10
|
spec.email = ["pablo@pablocantero.com"]
|
11
11
|
spec.description = spec.summary = %q{Shoryuken is a super efficient AWS SQS thread based message processor}
|
12
12
|
spec.homepage = "https://github.com/phstc/shoryuken"
|
13
|
-
spec.license = "
|
13
|
+
spec.license = "LGPL-3.0"
|
14
14
|
|
15
15
|
spec.files = `git ls-files -z`.split("\x0")
|
16
16
|
spec.executables = %w[shoryuken]
|
@@ -23,5 +23,5 @@ Gem::Specification.new do |spec|
|
|
23
23
|
spec.add_development_dependency "pry-byebug"
|
24
24
|
|
25
25
|
spec.add_dependency "aws-sdk-v1"
|
26
|
-
spec.add_dependency "celluloid", "~> 0.
|
26
|
+
spec.add_dependency "celluloid", "~> 0.16.0"
|
27
27
|
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Shoryuken::DefaultWorkerRegistry do
|
4
|
+
class RegistryTestWorker
|
5
|
+
include Shoryuken::Worker
|
6
|
+
|
7
|
+
shoryuken_options queue: 'registry-test'
|
8
|
+
|
9
|
+
def perform(sqs_msg, body); end
|
10
|
+
end
|
11
|
+
|
12
|
+
subject do
|
13
|
+
Shoryuken.worker_registry
|
14
|
+
end
|
15
|
+
|
16
|
+
before do
|
17
|
+
subject.register_worker 'registry-test', RegistryTestWorker
|
18
|
+
end
|
19
|
+
|
20
|
+
describe 'a registry containing workers is cleared' do
|
21
|
+
it 'removes all registrations' do
|
22
|
+
queue = 'some-other-queue'
|
23
|
+
|
24
|
+
registry = described_class.new
|
25
|
+
|
26
|
+
worker_class = Class.new do
|
27
|
+
include Shoryuken::Worker
|
28
|
+
|
29
|
+
shoryuken_options queue: queue
|
30
|
+
|
31
|
+
def perform(sqs_msg, body); end
|
32
|
+
end
|
33
|
+
|
34
|
+
registry.register_worker(queue, worker_class)
|
35
|
+
|
36
|
+
expect(registry.workers(queue)).to eq([worker_class])
|
37
|
+
|
38
|
+
registry.clear
|
39
|
+
|
40
|
+
expect(registry.workers(queue)).to eq([])
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
describe 'a registry with workers is handling messages' do
|
45
|
+
def build_message queue, explicit_worker = nil
|
46
|
+
attributes = {}
|
47
|
+
attributes['shoryuken_class'] = {
|
48
|
+
string_value: explicit_worker.to_s,
|
49
|
+
data_type: 'String' } if explicit_worker
|
50
|
+
|
51
|
+
double AWS::SQS::ReceivedMessage,
|
52
|
+
body: 'test',
|
53
|
+
message_attributes: attributes,
|
54
|
+
message_id: SecureRandom.uuid
|
55
|
+
end
|
56
|
+
|
57
|
+
context 'a batch of messages is being processed' do
|
58
|
+
it 'returns an instance of the worker registered for that queue' do
|
59
|
+
batch = [build_message('default', RegistryTestWorker)]
|
60
|
+
expect(subject.fetch_worker('default', batch)).to be_instance_of(TestWorker)
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
context 'a single message is being processed' do
|
65
|
+
context 'a worker class name is included in the message attributes' do
|
66
|
+
it 'returns an instance of that worker' do
|
67
|
+
message = build_message('default', RegistryTestWorker)
|
68
|
+
expect(subject.fetch_worker('default', message)).to be_instance_of(RegistryTestWorker)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
context 'a worker class name is not included in the message attributes' do
|
73
|
+
it 'returns an instance of the worker registered for that queue' do
|
74
|
+
message = build_message('default')
|
75
|
+
expect(subject.fetch_worker('default', message)).to be_instance_of(TestWorker)
|
76
|
+
|
77
|
+
message = build_message('registry-test')
|
78
|
+
expect(subject.fetch_worker('registry-test', message)).to be_instance_of(RegistryTestWorker)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -2,6 +2,12 @@ require 'spec_helper'
|
|
2
2
|
require 'shoryuken/manager'
|
3
3
|
|
4
4
|
describe Shoryuken::Manager do
|
5
|
+
subject do
|
6
|
+
condvar = double(:condvar)
|
7
|
+
allow(condvar).to receive(:signal).and_return(nil)
|
8
|
+
Shoryuken::Manager.new(condvar)
|
9
|
+
end
|
10
|
+
|
5
11
|
describe 'Auto Scaling' do
|
6
12
|
it 'decreases weight' do
|
7
13
|
queue1 = 'shoryuken'
|
@@ -12,11 +12,12 @@ describe Shoryuken::Processor do
|
|
12
12
|
|
13
13
|
before do
|
14
14
|
allow(manager).to receive(:async).and_return(manager)
|
15
|
+
allow(manager).to receive(:real_thread)
|
15
16
|
allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
|
16
17
|
end
|
17
18
|
|
18
19
|
describe '#process' do
|
19
|
-
it '
|
20
|
+
it 'parses the body into JSON' do
|
20
21
|
TestWorker.get_shoryuken_options['body_parser'] = :json
|
21
22
|
|
22
23
|
body = { 'test' => 'hi' }
|
@@ -28,7 +29,7 @@ describe Shoryuken::Processor do
|
|
28
29
|
subject.process(queue, sqs_msg)
|
29
30
|
end
|
30
31
|
|
31
|
-
it '
|
32
|
+
it 'parses the body calling the proc' do
|
32
33
|
TestWorker.get_shoryuken_options['body_parser'] = Proc.new { |sqs_msg| "*#{sqs_msg.body}*" }
|
33
34
|
|
34
35
|
expect_any_instance_of(TestWorker).to receive(:perform).with(sqs_msg, '*test*')
|
@@ -38,7 +39,7 @@ describe Shoryuken::Processor do
|
|
38
39
|
subject.process(queue, sqs_msg)
|
39
40
|
end
|
40
41
|
|
41
|
-
it '
|
42
|
+
it 'parses the body as text' do
|
42
43
|
TestWorker.get_shoryuken_options['body_parser'] = :text
|
43
44
|
|
44
45
|
body = 'test'
|
@@ -50,7 +51,7 @@ describe Shoryuken::Processor do
|
|
50
51
|
subject.process(queue, sqs_msg)
|
51
52
|
end
|
52
53
|
|
53
|
-
it '
|
54
|
+
it 'parses calling `.parse`' do
|
54
55
|
TestWorker.get_shoryuken_options['body_parser'] = JSON
|
55
56
|
|
56
57
|
body = { 'test' => 'hi' }
|
@@ -77,7 +78,7 @@ describe Shoryuken::Processor do
|
|
77
78
|
end
|
78
79
|
|
79
80
|
context 'when `object_type: nil`' do
|
80
|
-
it '
|
81
|
+
it 'parses the body as text' do
|
81
82
|
TestWorker.get_shoryuken_options['body_parser'] = nil
|
82
83
|
|
83
84
|
body = 'test'
|
@@ -91,6 +92,8 @@ describe Shoryuken::Processor do
|
|
91
92
|
end
|
92
93
|
|
93
94
|
context 'when custom middleware' do
|
95
|
+
let(:queue) { 'worker_called_middleware' }
|
96
|
+
|
94
97
|
class WorkerCalledMiddleware
|
95
98
|
def call(worker, queue, sqs_msg, body)
|
96
99
|
# called is defined with `allow(...).to receive(...)`
|
@@ -100,6 +103,14 @@ describe Shoryuken::Processor do
|
|
100
103
|
end
|
101
104
|
|
102
105
|
before do
|
106
|
+
class WorkerCalledMiddlewareWorker
|
107
|
+
include Shoryuken::Worker
|
108
|
+
|
109
|
+
shoryuken_options queue: 'worker_called_middleware'
|
110
|
+
|
111
|
+
def perform(sqs_msg, body); end
|
112
|
+
end
|
113
|
+
|
103
114
|
Shoryuken.configure_server do |config|
|
104
115
|
config.server_middleware do |chain|
|
105
116
|
chain.add WorkerCalledMiddleware
|
@@ -118,8 +129,8 @@ describe Shoryuken::Processor do
|
|
118
129
|
it 'invokes middleware' do
|
119
130
|
expect(manager).to receive(:processor_done).with(queue, subject)
|
120
131
|
|
121
|
-
expect_any_instance_of(
|
122
|
-
expect_any_instance_of(
|
132
|
+
expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:perform).with(sqs_msg, sqs_msg.body)
|
133
|
+
expect_any_instance_of(WorkerCalledMiddlewareWorker).to receive(:called).with(sqs_msg, queue)
|
123
134
|
|
124
135
|
subject.process(queue, sqs_msg)
|
125
136
|
end
|
@@ -158,7 +169,7 @@ describe Shoryuken::Processor do
|
|
158
169
|
} }
|
159
170
|
|
160
171
|
it 'performs without delete' do
|
161
|
-
Shoryuken.
|
172
|
+
Shoryuken.worker_registry.clear # unregister TestWorker
|
162
173
|
|
163
174
|
expect(manager).to receive(:processor_done).with(queue, subject)
|
164
175
|
|
@@ -170,4 +181,50 @@ describe Shoryuken::Processor do
|
|
170
181
|
end
|
171
182
|
end
|
172
183
|
end
|
184
|
+
|
185
|
+
describe '#auto_visibility_timeout' do
|
186
|
+
let(:heartbeat) { sqs_queue.visibility_timeout - 5 }
|
187
|
+
let(:visibility_timeout) { sqs_queue.visibility_timeout }
|
188
|
+
|
189
|
+
before do
|
190
|
+
TestWorker.get_shoryuken_options['auto_visibility_timeout'] = true
|
191
|
+
|
192
|
+
allow(sqs_queue).to receive(:visibility_timeout).and_return(15)
|
193
|
+
end
|
194
|
+
|
195
|
+
context 'when the worker takes a long time', slow: true do
|
196
|
+
it 'extends the message invisibility to prevent it from being dequeued concurrently' do
|
197
|
+
TestWorker.get_shoryuken_options['body_parser'] = Proc.new do |sqs_msg|
|
198
|
+
sleep visibility_timeout
|
199
|
+
'test'
|
200
|
+
end
|
201
|
+
|
202
|
+
expect(sqs_msg).to receive(:visibility_timeout=).with(visibility_timeout).once
|
203
|
+
expect(manager).to receive(:processor_done).with(queue, subject)
|
204
|
+
|
205
|
+
allow(sqs_msg).to receive(:body).and_return('test')
|
206
|
+
|
207
|
+
subject.process(queue, sqs_msg)
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
context 'when the worker takes a short time' do
|
212
|
+
it 'does not extend the message invisibility' do
|
213
|
+
expect(sqs_msg).to receive(:visibility_timeout=).never
|
214
|
+
expect(manager).to receive(:processor_done).with(queue, subject)
|
215
|
+
|
216
|
+
allow(sqs_msg).to receive(:body).and_return('test')
|
217
|
+
|
218
|
+
subject.process(queue, sqs_msg)
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
context 'when the worker fails' do
|
223
|
+
it 'does not extend the message invisibility' do
|
224
|
+
expect(sqs_msg).to receive(:visibility_timeout=).never
|
225
|
+
expect_any_instance_of(TestWorker).to receive(:perform).and_raise 'worker failed'
|
226
|
+
expect { subject.process(queue, sqs_msg) }.to raise_error
|
227
|
+
end
|
228
|
+
end
|
229
|
+
end
|
173
230
|
end
|
data/spec/shoryuken/util_spec.rb
CHANGED
@@ -14,4 +14,14 @@ describe 'Shoryuken::Util' do
|
|
14
14
|
expect(subject.unparse_queues(queues)).to eq([['queue1', 2], ['queue2', 1], ['queue3', 1], ['queue4', 3]])
|
15
15
|
end
|
16
16
|
end
|
17
|
+
|
18
|
+
describe '#worker_name' do
|
19
|
+
let(:sqs_msg) { double AWS::SQS::ReceivedMessage, id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e', message_attributes: {} }
|
20
|
+
|
21
|
+
it 'returns Shoryuken worker name' do
|
22
|
+
expect(subject.worker_name(TestWorker, sqs_msg)).to eq 'TestWorker'
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'returns ActiveJob worker name'
|
26
|
+
end
|
17
27
|
end
|