shoryuken 2.1.1 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.codeclimate.yml +3 -8
- data/.travis.yml +4 -0
- data/CHANGELOG.md +19 -0
- data/README.md +62 -15
- data/lib/shoryuken.rb +3 -1
- data/lib/shoryuken/environment_loader.rb +2 -2
- data/lib/shoryuken/fetcher.rb +16 -45
- data/lib/shoryuken/launcher.rb +2 -3
- data/lib/shoryuken/manager.rb +72 -110
- data/lib/shoryuken/polling.rb +206 -0
- data/lib/shoryuken/processor.rb +8 -7
- data/lib/shoryuken/version.rb +1 -1
- data/spec/shoryuken/fetcher_spec.rb +18 -52
- data/spec/shoryuken/manager_spec.rb +63 -94
- data/spec/shoryuken/middleware/server/auto_delete_spec.rb +2 -2
- data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +4 -4
- data/spec/shoryuken/polling_spec.rb +239 -0
- data/test_workers/endless_interruptive_worker.rb +41 -0
- data/test_workers/endless_uninterruptive_worker.rb +44 -0
- metadata +7 -2
@@ -55,8 +55,8 @@ describe Shoryuken::Middleware::Server::AutoDelete do
|
|
55
55
|
expect(sqs_queue).to_not receive(:delete_messages)
|
56
56
|
|
57
57
|
expect {
|
58
|
-
subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise '
|
59
|
-
}.to raise_error(
|
58
|
+
subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' }
|
59
|
+
}.to raise_error('failed')
|
60
60
|
end
|
61
61
|
end
|
62
62
|
end
|
@@ -46,7 +46,7 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
|
|
46
46
|
allow(sqs_msg).to receive(:queue){ sqs_queue }
|
47
47
|
expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 300)
|
48
48
|
|
49
|
-
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
|
49
|
+
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
|
50
50
|
end
|
51
51
|
|
52
52
|
it 'retries the job with exponential backoff' do
|
@@ -56,7 +56,7 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
|
|
56
56
|
allow(sqs_msg).to receive(:queue){ sqs_queue }
|
57
57
|
expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
|
58
58
|
|
59
|
-
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
|
59
|
+
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
|
60
60
|
end
|
61
61
|
|
62
62
|
it 'uses the last retry interval when :receive_count exceeds the size of :retry_intervals' do
|
@@ -66,7 +66,7 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
|
|
66
66
|
allow(sqs_msg).to receive(:queue){ sqs_queue }
|
67
67
|
expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
|
68
68
|
|
69
|
-
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
|
69
|
+
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
|
70
70
|
end
|
71
71
|
|
72
72
|
it 'limits the visibility timeout to 12 hours from receipt of message' do
|
@@ -75,7 +75,7 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
|
|
75
75
|
allow(sqs_msg).to receive(:queue){ sqs_queue }
|
76
76
|
expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 43198)
|
77
77
|
|
78
|
-
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
|
78
|
+
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
|
79
79
|
end
|
80
80
|
end
|
81
81
|
end
|
@@ -0,0 +1,239 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'shoryuken/polling'
|
3
|
+
|
4
|
+
describe Shoryuken::Polling::WeightedRoundRobin do
|
5
|
+
let(:queue1) { 'shoryuken' }
|
6
|
+
let(:queue2) { 'uppercut' }
|
7
|
+
let(:queues) { Array.new }
|
8
|
+
subject { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
|
9
|
+
|
10
|
+
describe '#next_queue' do
|
11
|
+
it 'cycles' do
|
12
|
+
# [shoryuken, 2]
|
13
|
+
# [uppercut, 1]
|
14
|
+
queues << queue1
|
15
|
+
queues << queue1
|
16
|
+
queues << queue2
|
17
|
+
|
18
|
+
expect(subject.next_queue).to eq(queue1)
|
19
|
+
expect(subject.next_queue).to eq(queue2)
|
20
|
+
expect(subject.next_queue).to eq(queue1)
|
21
|
+
end
|
22
|
+
|
23
|
+
it 'returns nil if there are no active queues' do
|
24
|
+
expect(subject.next_queue).to eq(nil)
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'unpauses queues whose pause is expired' do
|
28
|
+
# [shoryuken, 2]
|
29
|
+
# [uppercut, 1]
|
30
|
+
queues << queue1
|
31
|
+
queues << queue1
|
32
|
+
queues << queue2
|
33
|
+
|
34
|
+
allow(subject).to receive(:delay).and_return(10)
|
35
|
+
|
36
|
+
now = Time.now
|
37
|
+
allow(Time).to receive(:now).and_return(now)
|
38
|
+
|
39
|
+
# pause the first queue
|
40
|
+
subject.messages_found(queue1, 0)
|
41
|
+
expect(subject.next_queue).to eq(queue2)
|
42
|
+
|
43
|
+
now += 5
|
44
|
+
allow(Time).to receive(:now).and_return(now)
|
45
|
+
|
46
|
+
# pause the second queue
|
47
|
+
subject.messages_found(queue2, 0)
|
48
|
+
expect(subject.next_queue).to eq(nil)
|
49
|
+
|
50
|
+
# queue1 should be unpaused now
|
51
|
+
now += 6
|
52
|
+
allow(Time).to receive(:now).and_return(now)
|
53
|
+
expect(subject.next_queue).to eq(queue1)
|
54
|
+
|
55
|
+
# queue1 should be unpaused and added to the end of queues now
|
56
|
+
now += 6
|
57
|
+
allow(Time).to receive(:now).and_return(now)
|
58
|
+
expect(subject.next_queue).to eq(queue1)
|
59
|
+
expect(subject.next_queue).to eq(queue2)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
describe '#messages_found' do
|
64
|
+
it 'pauses a queue if there are no messages found' do
|
65
|
+
# [shoryuken, 2]
|
66
|
+
# [uppercut, 1]
|
67
|
+
queues << queue1
|
68
|
+
queues << queue1
|
69
|
+
queues << queue2
|
70
|
+
|
71
|
+
expect(subject).to receive(:pause).with(queue1).and_call_original
|
72
|
+
subject.messages_found(queue1, 0)
|
73
|
+
expect(subject.instance_variable_get(:@queues)).to eq([queue2])
|
74
|
+
end
|
75
|
+
|
76
|
+
it 'increased the weight if message is found' do
|
77
|
+
# [shoryuken, 2]
|
78
|
+
# [uppercut, 1]
|
79
|
+
queues << queue1
|
80
|
+
queues << queue1
|
81
|
+
queues << queue2
|
82
|
+
|
83
|
+
expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2])
|
84
|
+
subject.messages_found(queue1, 1)
|
85
|
+
expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
|
86
|
+
end
|
87
|
+
|
88
|
+
it 'respects the maximum queue weight' do
|
89
|
+
# [shoryuken, 2]
|
90
|
+
# [uppercut, 1]
|
91
|
+
queues << queue1
|
92
|
+
queues << queue1
|
93
|
+
queues << queue2
|
94
|
+
|
95
|
+
subject.messages_found(queue1, 1)
|
96
|
+
subject.messages_found(queue1, 1)
|
97
|
+
expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
describe Shoryuken::Polling::StrictPriority do
|
103
|
+
let(:queue1) { 'shoryuken' }
|
104
|
+
let(:queue2) { 'uppercut' }
|
105
|
+
let(:queue3) { 'other' }
|
106
|
+
let(:queues) { Array.new }
|
107
|
+
subject { Shoryuken::Polling::StrictPriority.new(queues) }
|
108
|
+
|
109
|
+
describe '#next_queue' do
|
110
|
+
it 'cycles when declared desc' do
|
111
|
+
# [shoryuken, 2]
|
112
|
+
# [uppercut, 1]
|
113
|
+
queues << queue1
|
114
|
+
queues << queue1
|
115
|
+
queues << queue2
|
116
|
+
|
117
|
+
expect(subject.next_queue).to eq(queue1)
|
118
|
+
expect(subject.next_queue).to eq(queue2)
|
119
|
+
expect(subject.next_queue).to eq(queue1)
|
120
|
+
expect(subject.next_queue).to eq(queue2)
|
121
|
+
end
|
122
|
+
|
123
|
+
it 'cycles when declared asc' do
|
124
|
+
# [uppercut, 1]
|
125
|
+
# [shoryuken, 2]
|
126
|
+
queues << queue2
|
127
|
+
queues << queue1
|
128
|
+
queues << queue1
|
129
|
+
|
130
|
+
expect(subject.next_queue).to eq(queue1)
|
131
|
+
expect(subject.next_queue).to eq(queue2)
|
132
|
+
expect(subject.next_queue).to eq(queue1)
|
133
|
+
expect(subject.next_queue).to eq(queue2)
|
134
|
+
end
|
135
|
+
|
136
|
+
it 'returns nil if there are no active queues' do
|
137
|
+
expect(subject.next_queue).to eq(nil)
|
138
|
+
end
|
139
|
+
|
140
|
+
it 'unpauses queues whose pause is expired' do
|
141
|
+
# [shoryuken, 3]
|
142
|
+
# [uppercut, 2]
|
143
|
+
# [other, 1]
|
144
|
+
queues << queue1
|
145
|
+
queues << queue1
|
146
|
+
queues << queue1
|
147
|
+
queues << queue2
|
148
|
+
queues << queue2
|
149
|
+
queues << queue3
|
150
|
+
|
151
|
+
allow(subject).to receive(:delay).and_return(10)
|
152
|
+
|
153
|
+
now = Time.now
|
154
|
+
allow(Time).to receive(:now).and_return(now)
|
155
|
+
|
156
|
+
# pause the second queue, see it loop between 1 and 3
|
157
|
+
subject.messages_found(queue2, 0)
|
158
|
+
expect(subject.next_queue).to eq(queue1)
|
159
|
+
expect(subject.next_queue).to eq(queue3)
|
160
|
+
expect(subject.next_queue).to eq(queue1)
|
161
|
+
|
162
|
+
now += 5
|
163
|
+
allow(Time).to receive(:now).and_return(now)
|
164
|
+
|
165
|
+
# pause the first queue, see it repeat 3
|
166
|
+
subject.messages_found(queue1, 0)
|
167
|
+
expect(subject.next_queue).to eq(queue3)
|
168
|
+
expect(subject.next_queue).to eq(queue3)
|
169
|
+
|
170
|
+
# pause the third queue, see it have nothing
|
171
|
+
subject.messages_found(queue3, 0)
|
172
|
+
expect(subject.next_queue).to eq(nil)
|
173
|
+
|
174
|
+
# unpause queue 2
|
175
|
+
now += 6
|
176
|
+
allow(Time).to receive(:now).and_return(now)
|
177
|
+
expect(subject.next_queue).to eq(queue2)
|
178
|
+
|
179
|
+
# unpause queues 1 and 3
|
180
|
+
now += 6
|
181
|
+
allow(Time).to receive(:now).and_return(now)
|
182
|
+
expect(subject.next_queue).to eq(queue1)
|
183
|
+
expect(subject.next_queue).to eq(queue2)
|
184
|
+
expect(subject.next_queue).to eq(queue3)
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
describe '#messages_found' do
|
189
|
+
it 'pauses a queue if there are no messages found' do
|
190
|
+
# [shoryuken, 2]
|
191
|
+
# [uppercut, 1]
|
192
|
+
queues << queue1
|
193
|
+
queues << queue1
|
194
|
+
queues << queue2
|
195
|
+
|
196
|
+
expect(subject.active_queues).to eq([[queue1, 2], [queue2, 1]])
|
197
|
+
expect(subject).to receive(:pause).with(queue1).and_call_original
|
198
|
+
subject.messages_found(queue1, 0)
|
199
|
+
expect(subject.active_queues).to eq([[queue2, 1]])
|
200
|
+
end
|
201
|
+
|
202
|
+
it 'continues to queue the highest priority queue if messages are found' do
|
203
|
+
# [shoryuken, 3]
|
204
|
+
# [uppercut, 2]
|
205
|
+
# [other, 1]
|
206
|
+
queues << queue1
|
207
|
+
queues << queue1
|
208
|
+
queues << queue1
|
209
|
+
queues << queue2
|
210
|
+
queues << queue2
|
211
|
+
queues << queue3
|
212
|
+
|
213
|
+
expect(subject.next_queue).to eq(queue1)
|
214
|
+
subject.messages_found(queue1, 1)
|
215
|
+
expect(subject.next_queue).to eq(queue1)
|
216
|
+
subject.messages_found(queue1, 1)
|
217
|
+
expect(subject.next_queue).to eq(queue1)
|
218
|
+
end
|
219
|
+
|
220
|
+
it 'resets the priorities if messages are found part way' do
|
221
|
+
# [shoryuken, 3]
|
222
|
+
# [uppercut, 2]
|
223
|
+
# [other, 1]
|
224
|
+
queues << queue1
|
225
|
+
queues << queue1
|
226
|
+
queues << queue1
|
227
|
+
queues << queue2
|
228
|
+
queues << queue2
|
229
|
+
queues << queue3
|
230
|
+
|
231
|
+
expect(subject.next_queue).to eq(queue1)
|
232
|
+
expect(subject.next_queue).to eq(queue2)
|
233
|
+
subject.messages_found(queue2, 1)
|
234
|
+
expect(subject.next_queue).to eq(queue1)
|
235
|
+
expect(subject.next_queue).to eq(queue2)
|
236
|
+
expect(subject.next_queue).to eq(queue3)
|
237
|
+
end
|
238
|
+
end
|
239
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
class EndlessInterruptiveWorker
|
2
|
+
include Shoryuken::Worker
|
3
|
+
|
4
|
+
# Usage:
|
5
|
+
# QUEUE="super-q"
|
6
|
+
# MAX_EXECUTION_TIME=2000 QUEUE=$QUEUE \
|
7
|
+
# bundle exec ./bin/shoryuken -r ./examples/endless_uninterruptive_worker.rb -q $QUEUE -c 8
|
8
|
+
|
9
|
+
class << self
|
10
|
+
def queue
|
11
|
+
ENV['QUEUE'] || 'default'
|
12
|
+
end
|
13
|
+
|
14
|
+
def max_execution_time
|
15
|
+
ENV["MAX_EXECUTION_TIME"] ? ENV["MAX_EXECUTION_TIME"].to_i : 100
|
16
|
+
end
|
17
|
+
|
18
|
+
def rng
|
19
|
+
@rng ||= Random.new
|
20
|
+
end
|
21
|
+
|
22
|
+
# returns a random number between 0 and 100
|
23
|
+
def random_number(hi = 1000)
|
24
|
+
(rng.rand * hi).to_i
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def perform(sqs_msg, body)
|
29
|
+
Shoryuken.logger.info("Received message: '#{body}'")
|
30
|
+
|
31
|
+
execution_ms = self.class.random_number(self.class.max_execution_time)
|
32
|
+
Shoryuken.logger.info("Going to sleep for #{execution_ms}ms")
|
33
|
+
|
34
|
+
new_body = "#{execution_ms}-" + body.to_s
|
35
|
+
sleep(execution_ms.to_f / 1000)
|
36
|
+
|
37
|
+
self.class.perform_async(new_body.slice(0, 512))
|
38
|
+
end
|
39
|
+
|
40
|
+
shoryuken_options queue: queue, auto_delete: true
|
41
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
class EndlessUninterruptiveWorker
|
2
|
+
include Shoryuken::Worker
|
3
|
+
|
4
|
+
# Usage:
|
5
|
+
# QUEUE="super-q"
|
6
|
+
# MAX_EXECUTION_TIME=2000 QUEUE=$QUEUE \
|
7
|
+
# bundle exec ./bin/shoryuken -r ./examples/endless_interruptive_worker.rb -q $QUEUE -c 8
|
8
|
+
|
9
|
+
class << self
|
10
|
+
def queue
|
11
|
+
ENV['QUEUE'] || 'default'
|
12
|
+
end
|
13
|
+
|
14
|
+
def max_execution_time
|
15
|
+
ENV["MAX_EXECUTION_TIME"] ? ENV["MAX_EXECUTION_TIME"].to_i : 100
|
16
|
+
end
|
17
|
+
|
18
|
+
def rng
|
19
|
+
@rng ||= Random.new
|
20
|
+
end
|
21
|
+
|
22
|
+
# returns a random number between 0 and 100
|
23
|
+
def random_number(hi = 1000)
|
24
|
+
(rng.rand * hi).to_i
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def perform(sqs_msg, body)
|
29
|
+
Shoryuken.logger.info("Received message: '#{body}'")
|
30
|
+
|
31
|
+
execution_ms = self.class.random_number(self.class.max_execution_time)
|
32
|
+
Shoryuken.logger.info("Going to burn metal for #{execution_ms}ms")
|
33
|
+
end_time = Time.now + execution_ms.to_f / 1000
|
34
|
+
while Time.now < end_time do
|
35
|
+
# burn metal
|
36
|
+
end
|
37
|
+
|
38
|
+
new_body = "#{execution_ms}-" + body.to_s
|
39
|
+
|
40
|
+
self.class.perform_async(new_body.slice(0, 512))
|
41
|
+
end
|
42
|
+
|
43
|
+
shoryuken_options queue: queue, auto_delete: true
|
44
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shoryuken
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.1.
|
4
|
+
version: 2.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pablo Cantero
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2016-12-
|
12
|
+
date: 2016-12-23 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|
@@ -164,6 +164,7 @@ files:
|
|
164
164
|
- lib/shoryuken/middleware/server/auto_extend_visibility.rb
|
165
165
|
- lib/shoryuken/middleware/server/exponential_backoff_retry.rb
|
166
166
|
- lib/shoryuken/middleware/server/timing.rb
|
167
|
+
- lib/shoryuken/polling.rb
|
167
168
|
- lib/shoryuken/processor.rb
|
168
169
|
- lib/shoryuken/queue.rb
|
169
170
|
- lib/shoryuken/sns_arn.rb
|
@@ -187,6 +188,7 @@ files:
|
|
187
188
|
- spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb
|
188
189
|
- spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb
|
189
190
|
- spec/shoryuken/middleware/server/timing_spec.rb
|
191
|
+
- spec/shoryuken/polling_spec.rb
|
190
192
|
- spec/shoryuken/processor_spec.rb
|
191
193
|
- spec/shoryuken/queue_spec.rb
|
192
194
|
- spec/shoryuken/sns_arn_spec.rb
|
@@ -196,6 +198,8 @@ files:
|
|
196
198
|
- spec/shoryuken_endpoint.yml
|
197
199
|
- spec/shoryuken_spec.rb
|
198
200
|
- spec/spec_helper.rb
|
201
|
+
- test_workers/endless_interruptive_worker.rb
|
202
|
+
- test_workers/endless_uninterruptive_worker.rb
|
199
203
|
homepage: https://github.com/phstc/shoryuken
|
200
204
|
licenses:
|
201
205
|
- LGPL-3.0
|
@@ -234,6 +238,7 @@ test_files:
|
|
234
238
|
- spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb
|
235
239
|
- spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb
|
236
240
|
- spec/shoryuken/middleware/server/timing_spec.rb
|
241
|
+
- spec/shoryuken/polling_spec.rb
|
237
242
|
- spec/shoryuken/processor_spec.rb
|
238
243
|
- spec/shoryuken/queue_spec.rb
|
239
244
|
- spec/shoryuken/sns_arn_spec.rb
|