shoryuken 2.1.1 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.codeclimate.yml +3 -8
- data/.travis.yml +4 -0
- data/CHANGELOG.md +19 -0
- data/README.md +62 -15
- data/lib/shoryuken.rb +3 -1
- data/lib/shoryuken/environment_loader.rb +2 -2
- data/lib/shoryuken/fetcher.rb +16 -45
- data/lib/shoryuken/launcher.rb +2 -3
- data/lib/shoryuken/manager.rb +72 -110
- data/lib/shoryuken/polling.rb +206 -0
- data/lib/shoryuken/processor.rb +8 -7
- data/lib/shoryuken/version.rb +1 -1
- data/spec/shoryuken/fetcher_spec.rb +18 -52
- data/spec/shoryuken/manager_spec.rb +63 -94
- data/spec/shoryuken/middleware/server/auto_delete_spec.rb +2 -2
- data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +4 -4
- data/spec/shoryuken/polling_spec.rb +239 -0
- data/test_workers/endless_interruptive_worker.rb +41 -0
- data/test_workers/endless_uninterruptive_worker.rb +44 -0
- metadata +7 -2
@@ -0,0 +1,206 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
module Polling
|
3
|
+
QueueConfiguration = Struct.new(:name, :options) do
|
4
|
+
def hash
|
5
|
+
name.hash
|
6
|
+
end
|
7
|
+
|
8
|
+
def ==(other)
|
9
|
+
case other
|
10
|
+
when String
|
11
|
+
if options.empty?
|
12
|
+
name == other
|
13
|
+
else
|
14
|
+
false
|
15
|
+
end
|
16
|
+
else
|
17
|
+
super
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
alias_method :eql?, :==
|
22
|
+
|
23
|
+
def to_s
|
24
|
+
if options.empty?
|
25
|
+
name
|
26
|
+
else
|
27
|
+
"#<QueueConfiguration #{name} options=#{options.inspect}>"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
class BaseStrategy
|
33
|
+
include Util
|
34
|
+
|
35
|
+
def next_queue
|
36
|
+
fail NotImplementedError
|
37
|
+
end
|
38
|
+
|
39
|
+
def messages_found(queue, messages_found)
|
40
|
+
fail NotImplementedError
|
41
|
+
end
|
42
|
+
|
43
|
+
def active_queues
|
44
|
+
fail NotImplementedError
|
45
|
+
end
|
46
|
+
|
47
|
+
def ==(other)
|
48
|
+
case other
|
49
|
+
when Array
|
50
|
+
@queues == other
|
51
|
+
else
|
52
|
+
if other.respond_to?(:active_queues)
|
53
|
+
active_queues == other.active_queues
|
54
|
+
else
|
55
|
+
false
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
def delay
|
63
|
+
Shoryuken.options[:delay].to_f
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
class WeightedRoundRobin < BaseStrategy
|
68
|
+
|
69
|
+
def initialize(queues)
|
70
|
+
@initial_queues = queues
|
71
|
+
@queues = queues.dup.uniq
|
72
|
+
@paused_queues = []
|
73
|
+
end
|
74
|
+
|
75
|
+
def next_queue
|
76
|
+
unpause_queues
|
77
|
+
queue = @queues.shift
|
78
|
+
return nil if queue.nil?
|
79
|
+
|
80
|
+
@queues << queue
|
81
|
+
QueueConfiguration.new(queue, {})
|
82
|
+
end
|
83
|
+
|
84
|
+
def messages_found(queue, messages_found)
|
85
|
+
if messages_found == 0
|
86
|
+
pause(queue)
|
87
|
+
return
|
88
|
+
end
|
89
|
+
|
90
|
+
maximum_weight = maximum_queue_weight(queue)
|
91
|
+
current_weight = current_queue_weight(queue)
|
92
|
+
if maximum_weight > current_weight
|
93
|
+
logger.info { "Increasing '#{queue}' weight to #{current_weight + 1}, max: #{maximum_weight}" }
|
94
|
+
@queues << queue
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def active_queues
|
99
|
+
unparse_queues(@queues)
|
100
|
+
end
|
101
|
+
|
102
|
+
private
|
103
|
+
|
104
|
+
def pause(queue)
|
105
|
+
return unless @queues.delete(queue)
|
106
|
+
@paused_queues << [Time.now + delay, queue]
|
107
|
+
logger.debug "Paused '#{queue}'"
|
108
|
+
end
|
109
|
+
|
110
|
+
def unpause_queues
|
111
|
+
return if @paused_queues.empty?
|
112
|
+
return if Time.now < @paused_queues.first[0]
|
113
|
+
pause = @paused_queues.shift
|
114
|
+
@queues << pause[1]
|
115
|
+
logger.debug "Unpaused '#{pause[1]}'"
|
116
|
+
end
|
117
|
+
|
118
|
+
def current_queue_weight(queue)
|
119
|
+
queue_weight(@queues, queue)
|
120
|
+
end
|
121
|
+
|
122
|
+
def maximum_queue_weight(queue)
|
123
|
+
queue_weight(@initial_queues, queue)
|
124
|
+
end
|
125
|
+
|
126
|
+
def queue_weight(queues, queue)
|
127
|
+
queues.count { |q| q == queue }
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
class StrictPriority < BaseStrategy
|
132
|
+
|
133
|
+
def initialize(queues)
|
134
|
+
# Priority ordering of the queues, highest priority first
|
135
|
+
@queues = queues
|
136
|
+
.group_by { |q| q }
|
137
|
+
.sort_by { |_, qs| -qs.count }
|
138
|
+
.map(&:first)
|
139
|
+
|
140
|
+
# Pause status of the queues, default to past time (unpaused)
|
141
|
+
@paused_until = queues
|
142
|
+
.each_with_object(Hash.new) { |queue, h| h[queue] = Time.at(0) }
|
143
|
+
|
144
|
+
# Start queues at 0
|
145
|
+
reset_next_queue
|
146
|
+
end
|
147
|
+
|
148
|
+
def next_queue
|
149
|
+
next_queue = next_active_queue
|
150
|
+
next_queue.nil? ? nil : QueueConfiguration.new(next_queue, {})
|
151
|
+
end
|
152
|
+
|
153
|
+
def messages_found(queue, messages_found)
|
154
|
+
if messages_found == 0
|
155
|
+
pause(queue)
|
156
|
+
else
|
157
|
+
reset_next_queue
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
def active_queues
|
162
|
+
@queues
|
163
|
+
.reverse
|
164
|
+
.map.with_index(1)
|
165
|
+
.reject { |q, _| queue_paused?(q) }
|
166
|
+
.reverse
|
167
|
+
end
|
168
|
+
|
169
|
+
private
|
170
|
+
|
171
|
+
def next_active_queue
|
172
|
+
reset_next_queue if queues_unpaused_since?
|
173
|
+
|
174
|
+
size = @queues.length
|
175
|
+
size.times do
|
176
|
+
queue = @queues[@next_queue_index]
|
177
|
+
@next_queue_index = (@next_queue_index + 1) % size
|
178
|
+
return queue unless queue_paused?(queue)
|
179
|
+
end
|
180
|
+
|
181
|
+
return nil
|
182
|
+
end
|
183
|
+
|
184
|
+
def queues_unpaused_since?
|
185
|
+
last = @last_unpause_check
|
186
|
+
now = @last_unpause_check = Time.now
|
187
|
+
|
188
|
+
last && @paused_until.values.any? { |t| t > last && t <= now }
|
189
|
+
end
|
190
|
+
|
191
|
+
def reset_next_queue
|
192
|
+
@next_queue_index = 0
|
193
|
+
end
|
194
|
+
|
195
|
+
def queue_paused?(queue)
|
196
|
+
@paused_until[queue] > Time.now
|
197
|
+
end
|
198
|
+
|
199
|
+
def pause(queue)
|
200
|
+
return unless delay > 0
|
201
|
+
@paused_until[queue] = Time.now + delay
|
202
|
+
logger.debug "Paused '#{queue}'"
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
206
|
+
end
|
data/lib/shoryuken/processor.rb
CHANGED
@@ -12,17 +12,18 @@ module Shoryuken
|
|
12
12
|
attr_accessor :proxy_id
|
13
13
|
|
14
14
|
def process(queue, sqs_msg)
|
15
|
-
@manager.async.real_thread(proxy_id, Thread.current)
|
16
|
-
|
17
15
|
worker = Shoryuken.worker_registry.fetch_worker(queue, sqs_msg)
|
16
|
+
body = get_body(worker.class, sqs_msg)
|
18
17
|
|
19
|
-
|
18
|
+
worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
|
19
|
+
worker.perform(sqs_msg, body)
|
20
|
+
end
|
20
21
|
|
21
|
-
|
22
|
-
|
23
|
-
end
|
22
|
+
@manager.async.processor_done(queue, current_actor)
|
23
|
+
end
|
24
24
|
|
25
|
-
|
25
|
+
def running_thread
|
26
|
+
Thread.current
|
26
27
|
end
|
27
28
|
|
28
29
|
private
|
data/lib/shoryuken/version.rb
CHANGED
@@ -3,69 +3,35 @@ require 'shoryuken/manager'
|
|
3
3
|
require 'shoryuken/fetcher'
|
4
4
|
|
5
5
|
describe Shoryuken::Fetcher do
|
6
|
-
let(:
|
7
|
-
let(:queue) { double Shoryuken::Queue }
|
6
|
+
let(:queue) { instance_double('Shoryuken::Queue') }
|
8
7
|
let(:queue_name) { 'default' }
|
8
|
+
let(:queue_config) { Shoryuken::Polling::QueueConfiguration.new(queue_name, {}) }
|
9
9
|
|
10
10
|
let(:sqs_msg) do
|
11
|
-
double
|
11
|
+
double(Shoryuken::Message,
|
12
12
|
queue_url: queue_name,
|
13
13
|
body: 'test',
|
14
|
-
message_id: '
|
15
|
-
|
16
|
-
|
17
|
-
subject { described_class.new(manager) }
|
18
|
-
|
19
|
-
before do
|
20
|
-
allow(manager).to receive(:async).and_return(manager)
|
21
|
-
allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
14
|
+
message_id: 'fc754df79cc24c4196ca5996a44b771e',
|
15
|
+
)
|
22
16
|
end
|
23
17
|
|
18
|
+
subject { described_class.new }
|
24
19
|
|
25
20
|
describe '#fetch' do
|
26
|
-
it 'calls
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
subject.fetch(queue_name, 1)
|
33
|
-
end
|
34
|
-
|
35
|
-
it 'assigns messages' do
|
36
|
-
allow(queue).to receive(:receive_messages).with(max_number_of_messages: 5, attribute_names: ['All'], message_attribute_names: ['All']).and_return(sqs_msg)
|
37
|
-
|
38
|
-
expect(manager).to receive(:rebalance_queue_weight!).with(queue_name)
|
39
|
-
expect(manager).to receive(:assign).with(queue_name, sqs_msg)
|
40
|
-
expect(manager).to receive(:dispatch)
|
41
|
-
|
42
|
-
subject.fetch(queue_name, 5)
|
21
|
+
it 'calls Shoryuken::Client to receive messages' do
|
22
|
+
expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
23
|
+
expect(queue).to receive(:receive_messages).
|
24
|
+
with(max_number_of_messages: 1, attribute_names: ['All'], message_attribute_names: ['All']).
|
25
|
+
and_return([])
|
26
|
+
subject.fetch(queue_config, 1)
|
43
27
|
end
|
44
28
|
|
45
|
-
it '
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
expect(manager).to receive(:assign).with(queue_name, [sqs_msg])
|
52
|
-
expect(manager).to receive(:dispatch)
|
53
|
-
|
54
|
-
subject.fetch(queue_name, 5)
|
55
|
-
end
|
56
|
-
|
57
|
-
context 'when worker not found' do
|
58
|
-
let(:queue_name) { 'notfound' }
|
59
|
-
|
60
|
-
it 'ignores batch' do
|
61
|
-
allow(queue).to receive(:receive_messages).with(max_number_of_messages: 5, attribute_names: ['All'], message_attribute_names: ['All']).and_return(sqs_msg)
|
62
|
-
|
63
|
-
expect(manager).to receive(:rebalance_queue_weight!).with(queue_name)
|
64
|
-
expect(manager).to receive(:assign).with(queue_name, sqs_msg)
|
65
|
-
expect(manager).to receive(:dispatch)
|
66
|
-
|
67
|
-
subject.fetch(queue_name, 5)
|
68
|
-
end
|
29
|
+
it 'maxes messages to receive to 10 (SQS limit)' do
|
30
|
+
allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
31
|
+
expect(queue).to receive(:receive_messages).
|
32
|
+
with(max_number_of_messages: 10, attribute_names: ['All'], message_attribute_names: ['All']).
|
33
|
+
and_return([])
|
34
|
+
subject.fetch(queue_config, 20)
|
69
35
|
end
|
70
36
|
end
|
71
37
|
end
|
@@ -1,11 +1,37 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
require 'shoryuken/manager'
|
3
3
|
|
4
|
+
RSpec::Matchers.define :queue_config_of do |expected|
|
5
|
+
match do |actual|
|
6
|
+
actual.name == expected
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
4
10
|
RSpec.describe Shoryuken::Manager do
|
5
|
-
|
11
|
+
let(:queue) { 'default' }
|
12
|
+
let(:queues) { [queue] }
|
13
|
+
let(:polling_strategy) { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
|
14
|
+
let(:fetcher) { Shoryuken::Fetcher.new }
|
15
|
+
let(:condvar) do
|
6
16
|
condvar = double(:condvar)
|
7
17
|
allow(condvar).to receive(:signal).and_return(nil)
|
8
|
-
|
18
|
+
condvar
|
19
|
+
end
|
20
|
+
let(:async_manager) { instance_double(described_class.name) }
|
21
|
+
let(:concurrency) { 1 }
|
22
|
+
|
23
|
+
subject { Shoryuken::Manager.new(condvar) }
|
24
|
+
|
25
|
+
before(:each) do
|
26
|
+
Shoryuken.options[:concurrency] = concurrency
|
27
|
+
subject.fetcher = fetcher
|
28
|
+
subject.polling_strategy = polling_strategy
|
29
|
+
allow_any_instance_of(described_class).to receive(:async).and_return(async_manager)
|
30
|
+
end
|
31
|
+
|
32
|
+
after(:each) do
|
33
|
+
Shoryuken.options[:concurrency] = 1
|
34
|
+
TestWorker.get_shoryuken_options['batch'] = false
|
9
35
|
end
|
10
36
|
|
11
37
|
describe 'Invalid concurrency setting' do
|
@@ -14,109 +40,52 @@ RSpec.describe Shoryuken::Manager do
|
|
14
40
|
expect { Shoryuken::Manager.new(nil) }
|
15
41
|
.to raise_error(ArgumentError, 'Concurrency value -1 is invalid, it needs to be a positive number')
|
16
42
|
end
|
17
|
-
|
18
43
|
end
|
19
44
|
|
20
|
-
describe '
|
21
|
-
it '
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
Shoryuken.queues.clear
|
26
|
-
# [shoryuken, 2]
|
27
|
-
# [uppercut, 1]
|
28
|
-
Shoryuken.queues << queue1
|
29
|
-
Shoryuken.queues << queue1
|
30
|
-
Shoryuken.queues << queue2
|
31
|
-
|
32
|
-
expect(subject.instance_variable_get('@queues')).to eq [queue1, queue2]
|
33
|
-
|
34
|
-
subject.pause_queue!(queue1)
|
35
|
-
|
36
|
-
expect(subject.instance_variable_get('@queues')).to eq [queue2]
|
45
|
+
describe '#dispatch' do
|
46
|
+
it 'pauses when there are no active queues' do
|
47
|
+
expect(polling_strategy).to receive(:next_queue).and_return(nil)
|
48
|
+
expect_any_instance_of(described_class).to receive(:after)
|
49
|
+
subject.dispatch
|
37
50
|
end
|
38
51
|
|
39
|
-
it '
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
# [uppercut, 1]
|
46
|
-
Shoryuken.queues << queue1
|
47
|
-
Shoryuken.queues << queue1
|
48
|
-
Shoryuken.queues << queue1
|
49
|
-
Shoryuken.queues << queue2
|
50
|
-
|
51
|
-
expect(subject.instance_variable_get('@queues')).to eq [queue1, queue2]
|
52
|
-
subject.pause_queue!(queue1)
|
53
|
-
expect(subject.instance_variable_get('@queues')).to eq [queue2]
|
54
|
-
|
55
|
-
subject.rebalance_queue_weight!(queue1)
|
56
|
-
expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1]
|
57
|
-
|
58
|
-
subject.rebalance_queue_weight!(queue1)
|
59
|
-
expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1, queue1]
|
60
|
-
|
61
|
-
subject.rebalance_queue_weight!(queue1)
|
62
|
-
expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1, queue1, queue1]
|
52
|
+
it 'calls dispatch_batch if worker wants batches' do
|
53
|
+
TestWorker.get_shoryuken_options['batch'] = true
|
54
|
+
expect_any_instance_of(described_class).to receive(:dispatch_batch).with(queue_config_of(queue))
|
55
|
+
expect_any_instance_of(described_class).to receive(:async).and_return(async_manager)
|
56
|
+
expect(async_manager).to receive(:dispatch)
|
57
|
+
subject.dispatch
|
63
58
|
end
|
64
59
|
|
65
|
-
it '
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
# [shoryuken, 2]
|
71
|
-
# [uppercut, 1]
|
72
|
-
Shoryuken.queues << queue1
|
73
|
-
Shoryuken.queues << queue1
|
74
|
-
Shoryuken.queues << queue2
|
75
|
-
|
76
|
-
Shoryuken.options[:delay] = 0.1
|
77
|
-
|
78
|
-
fetcher = double('Fetcher').as_null_object
|
79
|
-
subject.fetcher = fetcher
|
80
|
-
|
81
|
-
subject.pause_queue!(queue1)
|
82
|
-
expect(subject.instance_variable_get('@queues')).to eq [queue2]
|
83
|
-
|
84
|
-
sleep 0.5
|
85
|
-
|
86
|
-
expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1]
|
60
|
+
it 'calls dispatch_single_messages if worker wants single messages' do
|
61
|
+
expect_any_instance_of(described_class).to receive(:dispatch_single_messages).
|
62
|
+
with(queue_config_of(queue))
|
63
|
+
expect(async_manager).to receive(:dispatch)
|
64
|
+
subject.dispatch
|
87
65
|
end
|
88
66
|
end
|
89
67
|
|
90
|
-
describe '#
|
91
|
-
it '
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
Shoryuken.register_worker queue1, TestWorker
|
98
|
-
Shoryuken.register_worker queue2, TestWorker
|
99
|
-
|
100
|
-
Shoryuken.queues << queue1
|
101
|
-
Shoryuken.queues << queue2
|
102
|
-
|
103
|
-
expect(subject.send :next_queue).to eq queue1
|
104
|
-
expect(subject.send :next_queue).to eq queue2
|
68
|
+
describe '#dispatch_batch' do
|
69
|
+
it 'assings batch as a single message' do
|
70
|
+
q = polling_strategy.next_queue
|
71
|
+
messages = [1, 2, 3]
|
72
|
+
expect(fetcher).to receive(:fetch).with(q, described_class::BATCH_LIMIT).and_return(messages)
|
73
|
+
expect_any_instance_of(described_class).to receive(:assign).with(q.name, messages)
|
74
|
+
subject.send(:dispatch_batch, q)
|
105
75
|
end
|
76
|
+
end
|
106
77
|
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
expect(subject.send :next_queue).to eq queue2
|
119
|
-
expect(subject.send :next_queue).to eq queue2
|
78
|
+
describe '#dispatch_single_messages' do
|
79
|
+
let(:concurrency) { 3 }
|
80
|
+
|
81
|
+
it 'assings messages from batch one by one' do
|
82
|
+
q = polling_strategy.next_queue
|
83
|
+
messages = [1, 2, 3]
|
84
|
+
expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
|
85
|
+
expect_any_instance_of(described_class).to receive(:assign).with(q.name, 1)
|
86
|
+
expect_any_instance_of(described_class).to receive(:assign).with(q.name, 2)
|
87
|
+
expect_any_instance_of(described_class).to receive(:assign).with(q.name, 3)
|
88
|
+
subject.send(:dispatch_single_messages, q)
|
120
89
|
end
|
121
90
|
end
|
122
91
|
end
|