shoryuken 2.1.1 → 3.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.codeclimate.yml +5 -8
- data/.rubocop.yml +8 -2
- data/.travis.yml +6 -0
- data/CHANGELOG.md +167 -0
- data/Gemfile +2 -0
- data/README.md +16 -155
- data/Rakefile +0 -1
- data/bin/cli/base.rb +40 -0
- data/bin/cli/sqs.rb +204 -0
- data/bin/shoryuken +55 -9
- data/examples/default_worker.rb +1 -1
- data/lib/shoryuken/client.rb +3 -15
- data/lib/shoryuken/default_worker_registry.rb +10 -6
- data/lib/shoryuken/environment_loader.rb +60 -57
- data/lib/shoryuken/fetcher.rb +21 -51
- data/lib/shoryuken/launcher.rb +77 -23
- data/lib/shoryuken/logging.rb +0 -5
- data/lib/shoryuken/manager.rb +54 -205
- data/lib/shoryuken/message.rb +4 -13
- data/lib/shoryuken/middleware/chain.rb +1 -18
- data/lib/shoryuken/middleware/server/auto_delete.rb +3 -8
- data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +16 -17
- data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +37 -22
- data/lib/shoryuken/middleware/server/timing.rb +2 -2
- data/lib/shoryuken/options.rb +196 -0
- data/lib/shoryuken/polling/base.rb +67 -0
- data/lib/shoryuken/polling/strict_priority.rb +77 -0
- data/lib/shoryuken/polling/weighted_round_robin.rb +66 -0
- data/lib/shoryuken/processor.rb +27 -25
- data/lib/shoryuken/queue.rb +28 -9
- data/lib/shoryuken/runner.rb +131 -0
- data/lib/shoryuken/util.rb +1 -9
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker.rb +9 -1
- data/lib/shoryuken.rb +47 -157
- data/shoryuken.gemspec +7 -7
- data/spec/integration/launcher_spec.rb +15 -8
- data/spec/shoryuken/client_spec.rb +2 -45
- data/spec/shoryuken/default_worker_registry_spec.rb +12 -10
- data/spec/shoryuken/environment_loader_spec.rb +54 -0
- data/spec/shoryuken/fetcher_spec.rb +27 -46
- data/spec/shoryuken/manager_spec.rb +80 -93
- data/spec/shoryuken/middleware/chain_spec.rb +0 -24
- data/spec/shoryuken/middleware/server/auto_delete_spec.rb +2 -2
- data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb +7 -3
- data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +56 -33
- data/spec/shoryuken/middleware/server/timing_spec.rb +5 -3
- data/spec/shoryuken/options_spec.rb +100 -0
- data/spec/shoryuken/polling/strict_priority_spec.rb +140 -0
- data/spec/shoryuken/polling/weighted_round_robin_spec.rb +99 -0
- data/spec/shoryuken/processor_spec.rb +20 -37
- data/spec/shoryuken/queue_spec.rb +72 -26
- data/spec/shoryuken/{cli_spec.rb → runner_spec.rb} +11 -26
- data/spec/shoryuken_spec.rb +1 -48
- data/spec/spec_helper.rb +14 -20
- data/test_workers/endless_interruptive_worker.rb +41 -0
- data/test_workers/endless_uninterruptive_worker.rb +44 -0
- metadata +39 -34
- data/lib/shoryuken/aws_config.rb +0 -64
- data/lib/shoryuken/cli.rb +0 -215
- data/lib/shoryuken/sns_arn.rb +0 -27
- data/lib/shoryuken/topic.rb +0 -17
- data/spec/shoryuken/sns_arn_spec.rb +0 -42
- data/spec/shoryuken/topic_spec.rb +0 -32
- data/spec/shoryuken_endpoint.yml +0 -6
|
@@ -1,122 +1,109 @@
|
|
|
1
1
|
require 'spec_helper'
|
|
2
2
|
require 'shoryuken/manager'
|
|
3
3
|
|
|
4
|
-
RSpec.
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
allow(condvar).to receive(:signal).and_return(nil)
|
|
8
|
-
Shoryuken::Manager.new(condvar)
|
|
9
|
-
end
|
|
10
|
-
|
|
11
|
-
describe 'Invalid concurrency setting' do
|
|
12
|
-
it 'raises ArgumentError if concurrency is not positive number' do
|
|
13
|
-
Shoryuken.options[:concurrency] = -1
|
|
14
|
-
expect { Shoryuken::Manager.new(nil) }
|
|
15
|
-
.to raise_error(ArgumentError, 'Concurrency value -1 is invalid, it needs to be a positive number')
|
|
16
|
-
end
|
|
17
|
-
|
|
4
|
+
RSpec::Matchers.define :queue_config_of do |expected|
|
|
5
|
+
match do |actual|
|
|
6
|
+
actual.name == expected
|
|
18
7
|
end
|
|
8
|
+
end
|
|
19
9
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
10
|
+
RSpec.describe Shoryuken::Manager do
|
|
11
|
+
let(:queue) { 'default' }
|
|
12
|
+
let(:queues) { [queue] }
|
|
13
|
+
let(:polling_strategy) { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
|
|
14
|
+
let(:fetcher) { double Shoryuken::Fetcher }
|
|
15
|
+
let(:concurrency) { 1 }
|
|
16
|
+
let(:executor) { Concurrent::ImmediateExecutor.new }
|
|
24
17
|
|
|
25
|
-
|
|
26
|
-
# [shoryuken, 2]
|
|
27
|
-
# [uppercut, 1]
|
|
28
|
-
Shoryuken.queues << queue1
|
|
29
|
-
Shoryuken.queues << queue1
|
|
30
|
-
Shoryuken.queues << queue2
|
|
18
|
+
subject { Shoryuken::Manager.new(fetcher, polling_strategy, concurrency, executor) }
|
|
31
19
|
|
|
32
|
-
|
|
20
|
+
before do
|
|
21
|
+
allow(fetcher).to receive(:fetch).and_return([])
|
|
22
|
+
end
|
|
33
23
|
|
|
34
|
-
|
|
24
|
+
after do
|
|
25
|
+
Shoryuken.options[:concurrency] = 1
|
|
26
|
+
TestWorker.get_shoryuken_options['batch'] = false
|
|
27
|
+
end
|
|
35
28
|
|
|
36
|
-
|
|
29
|
+
describe '#stop' do
|
|
30
|
+
specify do
|
|
31
|
+
allow(subject).to receive(:running?).and_return(true, true, false)
|
|
32
|
+
expect(subject).to receive(:dispatch).once.and_call_original
|
|
33
|
+
expect(subject).to receive(:dispatch_loop).twice.and_call_original
|
|
34
|
+
subject.start
|
|
37
35
|
end
|
|
36
|
+
end
|
|
38
37
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
Shoryuken.queues.clear
|
|
44
|
-
# [shoryuken, 3]
|
|
45
|
-
# [uppercut, 1]
|
|
46
|
-
Shoryuken.queues << queue1
|
|
47
|
-
Shoryuken.queues << queue1
|
|
48
|
-
Shoryuken.queues << queue1
|
|
49
|
-
Shoryuken.queues << queue2
|
|
50
|
-
|
|
51
|
-
expect(subject.instance_variable_get('@queues')).to eq [queue1, queue2]
|
|
52
|
-
subject.pause_queue!(queue1)
|
|
53
|
-
expect(subject.instance_variable_get('@queues')).to eq [queue2]
|
|
54
|
-
|
|
55
|
-
subject.rebalance_queue_weight!(queue1)
|
|
56
|
-
expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1]
|
|
57
|
-
|
|
58
|
-
subject.rebalance_queue_weight!(queue1)
|
|
59
|
-
expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1, queue1]
|
|
60
|
-
|
|
61
|
-
subject.rebalance_queue_weight!(queue1)
|
|
62
|
-
expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1, queue1, queue1]
|
|
38
|
+
describe '#start' do
|
|
39
|
+
before do
|
|
40
|
+
# prevent dispatch loop
|
|
41
|
+
allow(subject).to receive(:running?).and_return(true, true, false)
|
|
63
42
|
end
|
|
64
43
|
|
|
65
|
-
it '
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
# [shoryuken, 2]
|
|
71
|
-
# [uppercut, 1]
|
|
72
|
-
Shoryuken.queues << queue1
|
|
73
|
-
Shoryuken.queues << queue1
|
|
74
|
-
Shoryuken.queues << queue2
|
|
75
|
-
|
|
76
|
-
Shoryuken.options[:delay] = 0.1
|
|
77
|
-
|
|
78
|
-
fetcher = double('Fetcher').as_null_object
|
|
79
|
-
subject.fetcher = fetcher
|
|
80
|
-
|
|
81
|
-
subject.pause_queue!(queue1)
|
|
82
|
-
expect(subject.instance_variable_get('@queues')).to eq [queue2]
|
|
44
|
+
it 'pauses when there are no active queues' do
|
|
45
|
+
expect(polling_strategy).to receive(:next_queue).and_return(nil)
|
|
46
|
+
expect(subject).to receive(:dispatch).and_call_original
|
|
47
|
+
subject.start
|
|
48
|
+
end
|
|
83
49
|
|
|
84
|
-
|
|
50
|
+
it 'calls dispatch_batch if worker wants batches' do
|
|
51
|
+
TestWorker.get_shoryuken_options['batch'] = true
|
|
52
|
+
expect(subject).to receive(:dispatch_batch).with(queue_config_of(queue))
|
|
53
|
+
subject.start
|
|
54
|
+
end
|
|
85
55
|
|
|
86
|
-
|
|
56
|
+
it 'calls dispatch_single_messages if worker wants single messages' do
|
|
57
|
+
expect(subject).to receive(:dispatch_single_messages).with(queue_config_of(queue))
|
|
58
|
+
subject.start
|
|
87
59
|
end
|
|
88
60
|
end
|
|
89
61
|
|
|
90
|
-
describe '#
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
Shoryuken.queues.clear
|
|
62
|
+
describe '#dispatch' do
|
|
63
|
+
before do
|
|
64
|
+
allow(subject).to receive(:running?).and_return(true, true, false)
|
|
65
|
+
end
|
|
96
66
|
|
|
97
|
-
|
|
98
|
-
|
|
67
|
+
specify do
|
|
68
|
+
message = ['test1']
|
|
69
|
+
messages = [message]
|
|
70
|
+
q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
|
|
99
71
|
|
|
100
|
-
|
|
101
|
-
|
|
72
|
+
expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
|
|
73
|
+
expect(subject).to receive(:fire_event).with(:dispatch)
|
|
74
|
+
expect(Shoryuken::Processor).to receive(:process).with(q, message)
|
|
75
|
+
expect(Shoryuken.logger).to_not receive(:info)
|
|
102
76
|
|
|
103
|
-
|
|
104
|
-
expect(subject.send :next_queue).to eq queue2
|
|
77
|
+
subject.send(:dispatch)
|
|
105
78
|
end
|
|
106
79
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
80
|
+
context 'when batch' do
|
|
81
|
+
specify do
|
|
82
|
+
messages = %w(test1 test2 test3)
|
|
83
|
+
q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
|
|
110
84
|
|
|
111
|
-
|
|
85
|
+
expect(fetcher).to receive(:fetch).with(q, described_class::BATCH_LIMIT).and_return(messages)
|
|
86
|
+
expect(subject).to receive(:fire_event).with(:dispatch)
|
|
87
|
+
allow(subject).to receive(:batched_queue?).with(q).and_return(true)
|
|
88
|
+
expect(Shoryuken::Processor).to receive(:process).with(q, messages)
|
|
89
|
+
expect(Shoryuken.logger).to_not receive(:info)
|
|
112
90
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
91
|
+
subject.send(:dispatch)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
117
95
|
|
|
118
|
-
|
|
119
|
-
|
|
96
|
+
describe '#dispatch_single_messages' do
|
|
97
|
+
let(:concurrency) { 3 }
|
|
98
|
+
|
|
99
|
+
it 'assings messages from batch one by one' do
|
|
100
|
+
q = polling_strategy.next_queue
|
|
101
|
+
messages = [1, 2, 3]
|
|
102
|
+
expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
|
|
103
|
+
expect_any_instance_of(described_class).to receive(:assign).with(q.name, 1)
|
|
104
|
+
expect_any_instance_of(described_class).to receive(:assign).with(q.name, 2)
|
|
105
|
+
expect_any_instance_of(described_class).to receive(:assign).with(q.name, 3)
|
|
106
|
+
subject.send(:dispatch_single_messages, q)
|
|
120
107
|
end
|
|
121
108
|
end
|
|
122
109
|
end
|
|
@@ -44,28 +44,4 @@ describe Shoryuken::Middleware::Chain do
|
|
|
44
44
|
expect(final_action).to eq nil
|
|
45
45
|
expect(recorder).to eq []
|
|
46
46
|
end
|
|
47
|
-
|
|
48
|
-
class DeprecatedMiddleware
|
|
49
|
-
def call(worker_instance, queue, sqs_msg)
|
|
50
|
-
@@success = true
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
def self.success?
|
|
54
|
-
!!@@success
|
|
55
|
-
end
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
it 'patches deprecated middleware' do
|
|
59
|
-
subject.clear
|
|
60
|
-
|
|
61
|
-
expect(Shoryuken.logger).to receive(:warn) do |&block|
|
|
62
|
-
expect(block.call).to eq('[DEPRECATION] DeprecatedMiddleware#call(worker_instance, queue, sqs_msg) is deprecated. Please use DeprecatedMiddleware#call(worker_instance, queue, sqs_msg, body)')
|
|
63
|
-
end
|
|
64
|
-
|
|
65
|
-
subject.add DeprecatedMiddleware
|
|
66
|
-
|
|
67
|
-
subject.invoke TestWorker, 'test', double('SQS msg', body: 'test'), 'test'
|
|
68
|
-
|
|
69
|
-
expect(DeprecatedMiddleware.success?).to eq true
|
|
70
|
-
end
|
|
71
47
|
end
|
|
@@ -55,8 +55,8 @@ describe Shoryuken::Middleware::Server::AutoDelete do
|
|
|
55
55
|
expect(sqs_queue).to_not receive(:delete_messages)
|
|
56
56
|
|
|
57
57
|
expect {
|
|
58
|
-
subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise '
|
|
59
|
-
}.to raise_error(
|
|
58
|
+
subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' }
|
|
59
|
+
}.to raise_error('failed')
|
|
60
60
|
end
|
|
61
61
|
end
|
|
62
62
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
require 'spec_helper'
|
|
2
2
|
|
|
3
|
-
describe Shoryuken::Middleware::Server::AutoExtendVisibility do
|
|
3
|
+
RSpec.describe Shoryuken::Middleware::Server::AutoExtendVisibility do
|
|
4
4
|
let(:queue) { 'default' }
|
|
5
5
|
let(:visibility_timeout) { 3 }
|
|
6
6
|
let(:extend_upfront) { 1 }
|
|
@@ -12,8 +12,6 @@ describe Shoryuken::Middleware::Server::AutoExtendVisibility do
|
|
|
12
12
|
|
|
13
13
|
# We need to run our worker inside actor context.
|
|
14
14
|
class Runner
|
|
15
|
-
include Celluloid
|
|
16
|
-
|
|
17
15
|
def run_and_sleep(worker, queue, sqs_msg, interval)
|
|
18
16
|
Shoryuken::Middleware::Server::AutoExtendVisibility.new.call(worker, queue, sqs_msg, sqs_msg.body) do
|
|
19
17
|
sleep interval
|
|
@@ -34,6 +32,12 @@ describe Shoryuken::Middleware::Server::AutoExtendVisibility do
|
|
|
34
32
|
stub_const('Shoryuken::Middleware::Server::AutoExtendVisibility::EXTEND_UPFRONT_SECONDS', extend_upfront)
|
|
35
33
|
end
|
|
36
34
|
|
|
35
|
+
context 'when batch worker' do
|
|
36
|
+
it 'yields' do
|
|
37
|
+
expect { |b| subject.call(TestWorker.new, nil, [], nil, &b) }.to yield_control
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
37
41
|
it 'extends message visibility if jobs takes a long time' do
|
|
38
42
|
TestWorker.get_shoryuken_options['auto_visibility_timeout'] = true
|
|
39
43
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
require 'spec_helper'
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
# rubocop:disable Metrics/BlockLength, Metrics/BlockDelimiters
|
|
4
|
+
RSpec.describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
|
|
4
5
|
let(:queue) { 'default' }
|
|
5
6
|
let(:sqs_queue) { double Shoryuken::Queue }
|
|
6
7
|
let(:sqs_msg) { double Shoryuken::Message, queue_url: queue, body: 'test', receipt_handle: SecureRandom.uuid,
|
|
@@ -10,8 +11,14 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
|
|
|
10
11
|
allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
|
|
11
12
|
end
|
|
12
13
|
|
|
13
|
-
context 'when
|
|
14
|
-
it '
|
|
14
|
+
context 'when batch worker' do
|
|
15
|
+
it 'yields' do
|
|
16
|
+
expect { |b| subject.call(TestWorker.new, nil, [], nil, &b) }.to yield_control
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
context 'when no exception' do
|
|
21
|
+
it 'does not retry' do
|
|
15
22
|
TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
|
|
16
23
|
|
|
17
24
|
expect(sqs_msg).not_to receive(:change_visibility)
|
|
@@ -20,62 +27,78 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
|
|
|
20
27
|
end
|
|
21
28
|
end
|
|
22
29
|
|
|
23
|
-
context 'when
|
|
30
|
+
context 'when an error' do
|
|
31
|
+
context "and retry_intervals isn't set" do
|
|
32
|
+
it 'does not retry' do
|
|
33
|
+
expect(sqs_msg).not_to receive(:change_visibility)
|
|
24
34
|
|
|
25
|
-
|
|
26
|
-
|
|
35
|
+
expect {
|
|
36
|
+
subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'Error' }
|
|
37
|
+
}.to raise_error(RuntimeError, 'Error')
|
|
38
|
+
end
|
|
39
|
+
end
|
|
27
40
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
41
|
+
context 'and retry_intervals is a lambda' do
|
|
42
|
+
it 'retries' do
|
|
43
|
+
TestWorker.get_shoryuken_options['retry_intervals'] = ->(_attempts) { 500 }
|
|
44
|
+
|
|
45
|
+
allow(sqs_msg).to receive(:queue) { sqs_queue }
|
|
46
|
+
expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 500)
|
|
47
|
+
|
|
48
|
+
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
|
|
49
|
+
end
|
|
31
50
|
end
|
|
32
51
|
|
|
33
|
-
|
|
34
|
-
|
|
52
|
+
context 'and retry_intervals is empty' do
|
|
53
|
+
it 'does not retry' do
|
|
54
|
+
TestWorker.get_shoryuken_options['retry_intervals'] = []
|
|
35
55
|
|
|
36
|
-
|
|
56
|
+
expect(sqs_msg).not_to receive(:change_visibility)
|
|
37
57
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
58
|
+
expect {
|
|
59
|
+
subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'Error' }
|
|
60
|
+
}.to raise_error(RuntimeError, 'Error')
|
|
61
|
+
end
|
|
41
62
|
end
|
|
42
63
|
|
|
43
|
-
it '
|
|
64
|
+
it 'uses first interval ' do
|
|
44
65
|
TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
|
|
45
66
|
|
|
46
|
-
allow(sqs_msg).to receive(:queue){ sqs_queue }
|
|
67
|
+
allow(sqs_msg).to receive(:queue) { sqs_queue }
|
|
47
68
|
expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 300)
|
|
48
69
|
|
|
49
|
-
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
|
|
70
|
+
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
|
|
50
71
|
end
|
|
51
72
|
|
|
52
|
-
it '
|
|
73
|
+
it 'uses matching interval' do
|
|
53
74
|
TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
|
|
54
75
|
|
|
55
|
-
allow(sqs_msg).to receive(:attributes){ {'ApproximateReceiveCount' => 2 } }
|
|
56
|
-
allow(sqs_msg).to receive(:queue){ sqs_queue }
|
|
76
|
+
allow(sqs_msg).to receive(:attributes) { { 'ApproximateReceiveCount' => 2 } }
|
|
77
|
+
allow(sqs_msg).to receive(:queue) { sqs_queue }
|
|
57
78
|
expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
|
|
58
79
|
|
|
59
|
-
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
|
|
80
|
+
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
|
|
60
81
|
end
|
|
61
82
|
|
|
62
|
-
|
|
63
|
-
|
|
83
|
+
context 'when attempts exceeds retry_intervals' do
|
|
84
|
+
it 'uses last interval' do
|
|
85
|
+
TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
|
|
64
86
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
87
|
+
allow(sqs_msg).to receive(:attributes) { { 'ApproximateReceiveCount' => 3 } }
|
|
88
|
+
allow(sqs_msg).to receive(:queue) { sqs_queue }
|
|
89
|
+
expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
|
|
68
90
|
|
|
69
|
-
|
|
91
|
+
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
|
|
92
|
+
end
|
|
70
93
|
end
|
|
71
94
|
|
|
72
|
-
it 'limits the visibility timeout to 12 hours
|
|
73
|
-
TestWorker.get_shoryuken_options['retry_intervals'] = [
|
|
95
|
+
it 'limits the visibility timeout to 12 hours' do
|
|
96
|
+
TestWorker.get_shoryuken_options['retry_intervals'] = [86_400]
|
|
74
97
|
|
|
75
|
-
allow(sqs_msg).to receive(:queue){ sqs_queue }
|
|
76
|
-
expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout:
|
|
98
|
+
allow(sqs_msg).to receive(:queue) { sqs_queue }
|
|
99
|
+
expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 43_198)
|
|
77
100
|
|
|
78
|
-
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
|
|
101
|
+
expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
|
|
79
102
|
end
|
|
80
103
|
end
|
|
81
104
|
end
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
require 'spec_helper'
|
|
2
2
|
|
|
3
|
-
describe Shoryuken::Middleware::Server::Timing do
|
|
3
|
+
RSpec.describe Shoryuken::Middleware::Server::Timing do
|
|
4
4
|
let(:queue) { 'default' }
|
|
5
5
|
let(:sqs_queue) { double Shoryuken::Queue, visibility_timeout: 60 }
|
|
6
6
|
|
|
7
7
|
let(:sqs_msg) do
|
|
8
|
-
double
|
|
8
|
+
double(
|
|
9
|
+
Shoryuken::Message,
|
|
9
10
|
queue_url: queue,
|
|
10
11
|
body: 'test',
|
|
11
12
|
message_id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e'
|
|
13
|
+
)
|
|
12
14
|
end
|
|
13
15
|
|
|
14
16
|
before do
|
|
@@ -28,7 +30,7 @@ describe Shoryuken::Middleware::Server::Timing do
|
|
|
28
30
|
|
|
29
31
|
context 'when exceeded the `visibility_timeout`' do
|
|
30
32
|
it 'logs exceeded' do
|
|
31
|
-
allow(subject).to receive(:elapsed).and_return(
|
|
33
|
+
allow(subject).to receive(:elapsed).and_return(120_000)
|
|
32
34
|
|
|
33
35
|
expect(Shoryuken.logger).to receive(:info) do |&block|
|
|
34
36
|
expect(block.call).to match(/started at/)
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
|
|
3
|
+
RSpec.describe Shoryuken::Options do
|
|
4
|
+
describe '.add_group' do
|
|
5
|
+
before do
|
|
6
|
+
Shoryuken.groups.clear
|
|
7
|
+
Shoryuken.add_group('group1', 25)
|
|
8
|
+
Shoryuken.add_group('group2', 25)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
specify do
|
|
12
|
+
described_class.add_queue('queue1', 1, 'group1')
|
|
13
|
+
described_class.add_queue('queue2', 2, 'group2')
|
|
14
|
+
|
|
15
|
+
expect(described_class.groups['group1'][:queues]).to eq(%w(queue1))
|
|
16
|
+
expect(described_class.groups['group2'][:queues]).to eq(%w(queue2 queue2))
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
describe '.ungrouped_queues' do
|
|
21
|
+
before do
|
|
22
|
+
Shoryuken.groups.clear
|
|
23
|
+
Shoryuken.add_group('group1', 25)
|
|
24
|
+
Shoryuken.add_group('group2', 25)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
specify do
|
|
28
|
+
described_class.add_queue('queue1', 1, 'group1')
|
|
29
|
+
described_class.add_queue('queue2', 2, 'group2')
|
|
30
|
+
|
|
31
|
+
expect(described_class.ungrouped_queues).to eq(%w(queue1 queue2 queue2))
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
describe '.sqs_client_receive_message_opts' do
|
|
36
|
+
before do
|
|
37
|
+
Shoryuken.sqs_client_receive_message_opts
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
specify do
|
|
41
|
+
Shoryuken.sqs_client_receive_message_opts = { test: 1 }
|
|
42
|
+
expect(Shoryuken.sqs_client_receive_message_opts).to eq('default' => { test: 1 })
|
|
43
|
+
|
|
44
|
+
Shoryuken.sqs_client_receive_message_opts['group1'] = { test: 2 }
|
|
45
|
+
|
|
46
|
+
expect(Shoryuken.sqs_client_receive_message_opts).to eq(
|
|
47
|
+
'default' => { test: 1 },
|
|
48
|
+
'group1' => { test: 2 },
|
|
49
|
+
)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
describe '.register_worker' do
|
|
54
|
+
it 'registers a worker' do
|
|
55
|
+
described_class.worker_registry.clear
|
|
56
|
+
described_class.register_worker('default', TestWorker)
|
|
57
|
+
expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
it 'registers a batchable worker' do
|
|
61
|
+
described_class.worker_registry.clear
|
|
62
|
+
TestWorker.get_shoryuken_options['batch'] = true
|
|
63
|
+
described_class.register_worker('default', TestWorker)
|
|
64
|
+
expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
it 'allows multiple workers' do
|
|
68
|
+
described_class.worker_registry.clear
|
|
69
|
+
described_class.register_worker('default', TestWorker)
|
|
70
|
+
expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
|
|
71
|
+
|
|
72
|
+
class Test2Worker
|
|
73
|
+
include Shoryuken::Worker
|
|
74
|
+
|
|
75
|
+
shoryuken_options queue: 'default'
|
|
76
|
+
|
|
77
|
+
def perform(sqs_msg, body); end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
expect(described_class.worker_registry.workers('default')).to eq([Test2Worker])
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
it 'raises an exception when mixing batchable with non batchable' do
|
|
84
|
+
described_class.worker_registry.clear
|
|
85
|
+
TestWorker.get_shoryuken_options['batch'] = true
|
|
86
|
+
described_class.register_worker('default', TestWorker)
|
|
87
|
+
|
|
88
|
+
expect {
|
|
89
|
+
class BatchableWorker
|
|
90
|
+
include Shoryuken::Worker
|
|
91
|
+
|
|
92
|
+
shoryuken_options queue: 'default', batch: true
|
|
93
|
+
|
|
94
|
+
def perform(sqs_msg, body); end
|
|
95
|
+
end
|
|
96
|
+
}.to raise_error("Could not register BatchableWorker for default, because TestWorker is already registered for this queue, " \
|
|
97
|
+
"and Shoryuken doesn't support a batchable worker for a queue with multiple workers")
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
|
|
3
|
+
RSpec.describe Shoryuken::Polling::StrictPriority do
|
|
4
|
+
let(:queue1) { 'shoryuken' }
|
|
5
|
+
let(:queue2) { 'uppercut' }
|
|
6
|
+
let(:queue3) { 'other' }
|
|
7
|
+
let(:queues) { Array.new }
|
|
8
|
+
subject { Shoryuken::Polling::StrictPriority.new(queues) }
|
|
9
|
+
|
|
10
|
+
describe '#next_queue' do
|
|
11
|
+
it 'cycles when declared desc' do
|
|
12
|
+
# [shoryuken, 2]
|
|
13
|
+
# [uppercut, 1]
|
|
14
|
+
queues << queue1
|
|
15
|
+
queues << queue1
|
|
16
|
+
queues << queue2
|
|
17
|
+
|
|
18
|
+
expect(subject.next_queue).to eq(queue1)
|
|
19
|
+
expect(subject.next_queue).to eq(queue2)
|
|
20
|
+
expect(subject.next_queue).to eq(queue1)
|
|
21
|
+
expect(subject.next_queue).to eq(queue2)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
it 'cycles when declared asc' do
|
|
25
|
+
# [uppercut, 1]
|
|
26
|
+
# [shoryuken, 2]
|
|
27
|
+
queues << queue2
|
|
28
|
+
queues << queue1
|
|
29
|
+
queues << queue1
|
|
30
|
+
|
|
31
|
+
expect(subject.next_queue).to eq(queue1)
|
|
32
|
+
expect(subject.next_queue).to eq(queue2)
|
|
33
|
+
expect(subject.next_queue).to eq(queue1)
|
|
34
|
+
expect(subject.next_queue).to eq(queue2)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
it 'returns nil if there are no active queues' do
|
|
38
|
+
expect(subject.next_queue).to eq(nil)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
it 'unpauses queues whose pause is expired' do
|
|
42
|
+
# [shoryuken, 3]
|
|
43
|
+
# [uppercut, 2]
|
|
44
|
+
# [other, 1]
|
|
45
|
+
queues << queue1
|
|
46
|
+
queues << queue1
|
|
47
|
+
queues << queue1
|
|
48
|
+
queues << queue2
|
|
49
|
+
queues << queue2
|
|
50
|
+
queues << queue3
|
|
51
|
+
|
|
52
|
+
allow(subject).to receive(:delay).and_return(10)
|
|
53
|
+
|
|
54
|
+
now = Time.now
|
|
55
|
+
allow(Time).to receive(:now).and_return(now)
|
|
56
|
+
|
|
57
|
+
# pause the second queue, see it loop between 1 and 3
|
|
58
|
+
subject.messages_found(queue2, 0)
|
|
59
|
+
expect(subject.next_queue).to eq(queue1)
|
|
60
|
+
expect(subject.next_queue).to eq(queue3)
|
|
61
|
+
expect(subject.next_queue).to eq(queue1)
|
|
62
|
+
|
|
63
|
+
now += 5
|
|
64
|
+
allow(Time).to receive(:now).and_return(now)
|
|
65
|
+
|
|
66
|
+
# pause the first queue, see it repeat 3
|
|
67
|
+
subject.messages_found(queue1, 0)
|
|
68
|
+
expect(subject.next_queue).to eq(queue3)
|
|
69
|
+
expect(subject.next_queue).to eq(queue3)
|
|
70
|
+
|
|
71
|
+
# pause the third queue, see it have nothing
|
|
72
|
+
subject.messages_found(queue3, 0)
|
|
73
|
+
expect(subject.next_queue).to eq(nil)
|
|
74
|
+
|
|
75
|
+
# unpause queue 2
|
|
76
|
+
now += 6
|
|
77
|
+
allow(Time).to receive(:now).and_return(now)
|
|
78
|
+
expect(subject.next_queue).to eq(queue2)
|
|
79
|
+
|
|
80
|
+
# unpause queues 1 and 3
|
|
81
|
+
now += 6
|
|
82
|
+
allow(Time).to receive(:now).and_return(now)
|
|
83
|
+
expect(subject.next_queue).to eq(queue1)
|
|
84
|
+
expect(subject.next_queue).to eq(queue2)
|
|
85
|
+
expect(subject.next_queue).to eq(queue3)
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
describe '#messages_found' do
|
|
90
|
+
it 'pauses a queue if there are no messages found' do
|
|
91
|
+
# [shoryuken, 2]
|
|
92
|
+
# [uppercut, 1]
|
|
93
|
+
queues << queue1
|
|
94
|
+
queues << queue1
|
|
95
|
+
queues << queue2
|
|
96
|
+
|
|
97
|
+
expect(subject.active_queues).to eq([[queue1, 2], [queue2, 1]])
|
|
98
|
+
expect(subject).to receive(:pause).with(queue1).and_call_original
|
|
99
|
+
subject.messages_found(queue1, 0)
|
|
100
|
+
expect(subject.active_queues).to eq([[queue2, 1]])
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
it 'continues to queue the highest priority queue if messages are found' do
|
|
104
|
+
# [shoryuken, 3]
|
|
105
|
+
# [uppercut, 2]
|
|
106
|
+
# [other, 1]
|
|
107
|
+
queues << queue1
|
|
108
|
+
queues << queue1
|
|
109
|
+
queues << queue1
|
|
110
|
+
queues << queue2
|
|
111
|
+
queues << queue2
|
|
112
|
+
queues << queue3
|
|
113
|
+
|
|
114
|
+
expect(subject.next_queue).to eq(queue1)
|
|
115
|
+
subject.messages_found(queue1, 1)
|
|
116
|
+
expect(subject.next_queue).to eq(queue1)
|
|
117
|
+
subject.messages_found(queue1, 1)
|
|
118
|
+
expect(subject.next_queue).to eq(queue1)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
it 'resets the priorities if messages are found part way' do
|
|
122
|
+
# [shoryuken, 3]
|
|
123
|
+
# [uppercut, 2]
|
|
124
|
+
# [other, 1]
|
|
125
|
+
queues << queue1
|
|
126
|
+
queues << queue1
|
|
127
|
+
queues << queue1
|
|
128
|
+
queues << queue2
|
|
129
|
+
queues << queue2
|
|
130
|
+
queues << queue3
|
|
131
|
+
|
|
132
|
+
expect(subject.next_queue).to eq(queue1)
|
|
133
|
+
expect(subject.next_queue).to eq(queue2)
|
|
134
|
+
subject.messages_found(queue2, 1)
|
|
135
|
+
expect(subject.next_queue).to eq(queue1)
|
|
136
|
+
expect(subject.next_queue).to eq(queue2)
|
|
137
|
+
expect(subject.next_queue).to eq(queue3)
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
end
|