shoryuken 2.1.1 → 2.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -55,8 +55,8 @@ describe Shoryuken::Middleware::Server::AutoDelete do
55
55
  expect(sqs_queue).to_not receive(:delete_messages)
56
56
 
57
57
  expect {
58
- subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'Error' }
59
- }.to raise_error(RuntimeError, 'Error')
58
+ subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' }
59
+ }.to raise_error('failed')
60
60
  end
61
61
  end
62
62
  end
@@ -46,7 +46,7 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
46
46
  allow(sqs_msg).to receive(:queue){ sqs_queue }
47
47
  expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 300)
48
48
 
49
- expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
49
+ expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
50
50
  end
51
51
 
52
52
  it 'retries the job with exponential backoff' do
@@ -56,7 +56,7 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
56
56
  allow(sqs_msg).to receive(:queue){ sqs_queue }
57
57
  expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
58
58
 
59
- expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
59
+ expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
60
60
  end
61
61
 
62
62
  it 'uses the last retry interval when :receive_count exceeds the size of :retry_intervals' do
@@ -66,7 +66,7 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
66
66
  allow(sqs_msg).to receive(:queue){ sqs_queue }
67
67
  expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
68
68
 
69
- expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
69
+ expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
70
70
  end
71
71
 
72
72
  it 'limits the visibility timeout to 12 hours from receipt of message' do
@@ -75,7 +75,7 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
75
75
  allow(sqs_msg).to receive(:queue){ sqs_queue }
76
76
  expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 43198)
77
77
 
78
- expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
78
+ expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
79
79
  end
80
80
  end
81
81
  end
@@ -0,0 +1,239 @@
1
+ require 'spec_helper'
2
+ require 'shoryuken/polling'
3
+
4
+ describe Shoryuken::Polling::WeightedRoundRobin do
5
+ let(:queue1) { 'shoryuken' }
6
+ let(:queue2) { 'uppercut' }
7
+ let(:queues) { Array.new }
8
+ subject { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
9
+
10
+ describe '#next_queue' do
11
+ it 'cycles' do
12
+ # [shoryuken, 2]
13
+ # [uppercut, 1]
14
+ queues << queue1
15
+ queues << queue1
16
+ queues << queue2
17
+
18
+ expect(subject.next_queue).to eq(queue1)
19
+ expect(subject.next_queue).to eq(queue2)
20
+ expect(subject.next_queue).to eq(queue1)
21
+ end
22
+
23
+ it 'returns nil if there are no active queues' do
24
+ expect(subject.next_queue).to eq(nil)
25
+ end
26
+
27
+ it 'unpauses queues whose pause is expired' do
28
+ # [shoryuken, 2]
29
+ # [uppercut, 1]
30
+ queues << queue1
31
+ queues << queue1
32
+ queues << queue2
33
+
34
+ allow(subject).to receive(:delay).and_return(10)
35
+
36
+ now = Time.now
37
+ allow(Time).to receive(:now).and_return(now)
38
+
39
+ # pause the first queue
40
+ subject.messages_found(queue1, 0)
41
+ expect(subject.next_queue).to eq(queue2)
42
+
43
+ now += 5
44
+ allow(Time).to receive(:now).and_return(now)
45
+
46
+ # pause the second queue
47
+ subject.messages_found(queue2, 0)
48
+ expect(subject.next_queue).to eq(nil)
49
+
50
+ # queue1 should be unpaused now
51
+ now += 6
52
+ allow(Time).to receive(:now).and_return(now)
53
+ expect(subject.next_queue).to eq(queue1)
54
+
55
+ # queue1 should be unpaused and added to the end of queues now
56
+ now += 6
57
+ allow(Time).to receive(:now).and_return(now)
58
+ expect(subject.next_queue).to eq(queue1)
59
+ expect(subject.next_queue).to eq(queue2)
60
+ end
61
+ end
62
+
63
+ describe '#messages_found' do
64
+ it 'pauses a queue if there are no messages found' do
65
+ # [shoryuken, 2]
66
+ # [uppercut, 1]
67
+ queues << queue1
68
+ queues << queue1
69
+ queues << queue2
70
+
71
+ expect(subject).to receive(:pause).with(queue1).and_call_original
72
+ subject.messages_found(queue1, 0)
73
+ expect(subject.instance_variable_get(:@queues)).to eq([queue2])
74
+ end
75
+
76
+ it 'increased the weight if message is found' do
77
+ # [shoryuken, 2]
78
+ # [uppercut, 1]
79
+ queues << queue1
80
+ queues << queue1
81
+ queues << queue2
82
+
83
+ expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2])
84
+ subject.messages_found(queue1, 1)
85
+ expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
86
+ end
87
+
88
+ it 'respects the maximum queue weight' do
89
+ # [shoryuken, 2]
90
+ # [uppercut, 1]
91
+ queues << queue1
92
+ queues << queue1
93
+ queues << queue2
94
+
95
+ subject.messages_found(queue1, 1)
96
+ subject.messages_found(queue1, 1)
97
+ expect(subject.instance_variable_get(:@queues)).to eq([queue1, queue2, queue1])
98
+ end
99
+ end
100
+ end
101
+
102
+ describe Shoryuken::Polling::StrictPriority do
103
+ let(:queue1) { 'shoryuken' }
104
+ let(:queue2) { 'uppercut' }
105
+ let(:queue3) { 'other' }
106
+ let(:queues) { Array.new }
107
+ subject { Shoryuken::Polling::StrictPriority.new(queues) }
108
+
109
+ describe '#next_queue' do
110
+ it 'cycles when declared desc' do
111
+ # [shoryuken, 2]
112
+ # [uppercut, 1]
113
+ queues << queue1
114
+ queues << queue1
115
+ queues << queue2
116
+
117
+ expect(subject.next_queue).to eq(queue1)
118
+ expect(subject.next_queue).to eq(queue2)
119
+ expect(subject.next_queue).to eq(queue1)
120
+ expect(subject.next_queue).to eq(queue2)
121
+ end
122
+
123
+ it 'cycles when declared asc' do
124
+ # [uppercut, 1]
125
+ # [shoryuken, 2]
126
+ queues << queue2
127
+ queues << queue1
128
+ queues << queue1
129
+
130
+ expect(subject.next_queue).to eq(queue1)
131
+ expect(subject.next_queue).to eq(queue2)
132
+ expect(subject.next_queue).to eq(queue1)
133
+ expect(subject.next_queue).to eq(queue2)
134
+ end
135
+
136
+ it 'returns nil if there are no active queues' do
137
+ expect(subject.next_queue).to eq(nil)
138
+ end
139
+
140
+ it 'unpauses queues whose pause is expired' do
141
+ # [shoryuken, 3]
142
+ # [uppercut, 2]
143
+ # [other, 1]
144
+ queues << queue1
145
+ queues << queue1
146
+ queues << queue1
147
+ queues << queue2
148
+ queues << queue2
149
+ queues << queue3
150
+
151
+ allow(subject).to receive(:delay).and_return(10)
152
+
153
+ now = Time.now
154
+ allow(Time).to receive(:now).and_return(now)
155
+
156
+ # pause the second queue, see it loop between 1 and 3
157
+ subject.messages_found(queue2, 0)
158
+ expect(subject.next_queue).to eq(queue1)
159
+ expect(subject.next_queue).to eq(queue3)
160
+ expect(subject.next_queue).to eq(queue1)
161
+
162
+ now += 5
163
+ allow(Time).to receive(:now).and_return(now)
164
+
165
+ # pause the first queue, see it repeat 3
166
+ subject.messages_found(queue1, 0)
167
+ expect(subject.next_queue).to eq(queue3)
168
+ expect(subject.next_queue).to eq(queue3)
169
+
170
+ # pause the third queue, see it have nothing
171
+ subject.messages_found(queue3, 0)
172
+ expect(subject.next_queue).to eq(nil)
173
+
174
+ # unpause queue 2
175
+ now += 6
176
+ allow(Time).to receive(:now).and_return(now)
177
+ expect(subject.next_queue).to eq(queue2)
178
+
179
+ # unpause queues 1 and 3
180
+ now += 6
181
+ allow(Time).to receive(:now).and_return(now)
182
+ expect(subject.next_queue).to eq(queue1)
183
+ expect(subject.next_queue).to eq(queue2)
184
+ expect(subject.next_queue).to eq(queue3)
185
+ end
186
+ end
187
+
188
+ describe '#messages_found' do
189
+ it 'pauses a queue if there are no messages found' do
190
+ # [shoryuken, 2]
191
+ # [uppercut, 1]
192
+ queues << queue1
193
+ queues << queue1
194
+ queues << queue2
195
+
196
+ expect(subject.active_queues).to eq([[queue1, 2], [queue2, 1]])
197
+ expect(subject).to receive(:pause).with(queue1).and_call_original
198
+ subject.messages_found(queue1, 0)
199
+ expect(subject.active_queues).to eq([[queue2, 1]])
200
+ end
201
+
202
+ it 'continues to queue the highest priority queue if messages are found' do
203
+ # [shoryuken, 3]
204
+ # [uppercut, 2]
205
+ # [other, 1]
206
+ queues << queue1
207
+ queues << queue1
208
+ queues << queue1
209
+ queues << queue2
210
+ queues << queue2
211
+ queues << queue3
212
+
213
+ expect(subject.next_queue).to eq(queue1)
214
+ subject.messages_found(queue1, 1)
215
+ expect(subject.next_queue).to eq(queue1)
216
+ subject.messages_found(queue1, 1)
217
+ expect(subject.next_queue).to eq(queue1)
218
+ end
219
+
220
+ it 'resets the priorities if messages are found part way' do
221
+ # [shoryuken, 3]
222
+ # [uppercut, 2]
223
+ # [other, 1]
224
+ queues << queue1
225
+ queues << queue1
226
+ queues << queue1
227
+ queues << queue2
228
+ queues << queue2
229
+ queues << queue3
230
+
231
+ expect(subject.next_queue).to eq(queue1)
232
+ expect(subject.next_queue).to eq(queue2)
233
+ subject.messages_found(queue2, 1)
234
+ expect(subject.next_queue).to eq(queue1)
235
+ expect(subject.next_queue).to eq(queue2)
236
+ expect(subject.next_queue).to eq(queue3)
237
+ end
238
+ end
239
+ end
@@ -0,0 +1,41 @@
1
+ class EndlessInterruptiveWorker
2
+ include Shoryuken::Worker
3
+
4
+ # Usage:
5
+ # QUEUE="super-q"
6
+ # MAX_EXECUTION_TIME=2000 QUEUE=$QUEUE \
7
+ # bundle exec ./bin/shoryuken -r ./examples/endless_uninterruptive_worker.rb -q $QUEUE -c 8
8
+
9
+ class << self
10
+ def queue
11
+ ENV['QUEUE'] || 'default'
12
+ end
13
+
14
+ def max_execution_time
15
+ ENV["MAX_EXECUTION_TIME"] ? ENV["MAX_EXECUTION_TIME"].to_i : 100
16
+ end
17
+
18
+ def rng
19
+ @rng ||= Random.new
20
+ end
21
+
22
+ # returns a random number between 0 and 100
23
+ def random_number(hi = 1000)
24
+ (rng.rand * hi).to_i
25
+ end
26
+ end
27
+
28
+ def perform(sqs_msg, body)
29
+ Shoryuken.logger.info("Received message: '#{body}'")
30
+
31
+ execution_ms = self.class.random_number(self.class.max_execution_time)
32
+ Shoryuken.logger.info("Going to sleep for #{execution_ms}ms")
33
+
34
+ new_body = "#{execution_ms}-" + body.to_s
35
+ sleep(execution_ms.to_f / 1000)
36
+
37
+ self.class.perform_async(new_body.slice(0, 512))
38
+ end
39
+
40
+ shoryuken_options queue: queue, auto_delete: true
41
+ end
@@ -0,0 +1,44 @@
1
+ class EndlessUninterruptiveWorker
2
+ include Shoryuken::Worker
3
+
4
+ # Usage:
5
+ # QUEUE="super-q"
6
+ # MAX_EXECUTION_TIME=2000 QUEUE=$QUEUE \
7
+ # bundle exec ./bin/shoryuken -r ./examples/endless_interruptive_worker.rb -q $QUEUE -c 8
8
+
9
+ class << self
10
+ def queue
11
+ ENV['QUEUE'] || 'default'
12
+ end
13
+
14
+ def max_execution_time
15
+ ENV["MAX_EXECUTION_TIME"] ? ENV["MAX_EXECUTION_TIME"].to_i : 100
16
+ end
17
+
18
+ def rng
19
+ @rng ||= Random.new
20
+ end
21
+
22
+ # returns a random number between 0 and 100
23
+ def random_number(hi = 1000)
24
+ (rng.rand * hi).to_i
25
+ end
26
+ end
27
+
28
+ def perform(sqs_msg, body)
29
+ Shoryuken.logger.info("Received message: '#{body}'")
30
+
31
+ execution_ms = self.class.random_number(self.class.max_execution_time)
32
+ Shoryuken.logger.info("Going to burn metal for #{execution_ms}ms")
33
+ end_time = Time.now + execution_ms.to_f / 1000
34
+ while Time.now < end_time do
35
+ # burn metal
36
+ end
37
+
38
+ new_body = "#{execution_ms}-" + body.to_s
39
+
40
+ self.class.perform_async(new_body.slice(0, 512))
41
+ end
42
+
43
+ shoryuken_options queue: queue, auto_delete: true
44
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: shoryuken
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.1
4
+ version: 2.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Pablo Cantero
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2016-12-05 00:00:00.000000000 Z
12
+ date: 2016-12-23 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: bundler
@@ -164,6 +164,7 @@ files:
164
164
  - lib/shoryuken/middleware/server/auto_extend_visibility.rb
165
165
  - lib/shoryuken/middleware/server/exponential_backoff_retry.rb
166
166
  - lib/shoryuken/middleware/server/timing.rb
167
+ - lib/shoryuken/polling.rb
167
168
  - lib/shoryuken/processor.rb
168
169
  - lib/shoryuken/queue.rb
169
170
  - lib/shoryuken/sns_arn.rb
@@ -187,6 +188,7 @@ files:
187
188
  - spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb
188
189
  - spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb
189
190
  - spec/shoryuken/middleware/server/timing_spec.rb
191
+ - spec/shoryuken/polling_spec.rb
190
192
  - spec/shoryuken/processor_spec.rb
191
193
  - spec/shoryuken/queue_spec.rb
192
194
  - spec/shoryuken/sns_arn_spec.rb
@@ -196,6 +198,8 @@ files:
196
198
  - spec/shoryuken_endpoint.yml
197
199
  - spec/shoryuken_spec.rb
198
200
  - spec/spec_helper.rb
201
+ - test_workers/endless_interruptive_worker.rb
202
+ - test_workers/endless_uninterruptive_worker.rb
199
203
  homepage: https://github.com/phstc/shoryuken
200
204
  licenses:
201
205
  - LGPL-3.0
@@ -234,6 +238,7 @@ test_files:
234
238
  - spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb
235
239
  - spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb
236
240
  - spec/shoryuken/middleware/server/timing_spec.rb
241
+ - spec/shoryuken/polling_spec.rb
237
242
  - spec/shoryuken/processor_spec.rb
238
243
  - spec/shoryuken/queue_spec.rb
239
244
  - spec/shoryuken/sns_arn_spec.rb