shoryuken 2.1.1 → 3.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +5 -8
  3. data/.rubocop.yml +8 -2
  4. data/.travis.yml +6 -0
  5. data/CHANGELOG.md +167 -0
  6. data/Gemfile +2 -0
  7. data/README.md +16 -155
  8. data/Rakefile +0 -1
  9. data/bin/cli/base.rb +40 -0
  10. data/bin/cli/sqs.rb +204 -0
  11. data/bin/shoryuken +55 -9
  12. data/examples/default_worker.rb +1 -1
  13. data/lib/shoryuken/client.rb +3 -15
  14. data/lib/shoryuken/default_worker_registry.rb +10 -6
  15. data/lib/shoryuken/environment_loader.rb +60 -57
  16. data/lib/shoryuken/fetcher.rb +21 -51
  17. data/lib/shoryuken/launcher.rb +77 -23
  18. data/lib/shoryuken/logging.rb +0 -5
  19. data/lib/shoryuken/manager.rb +54 -205
  20. data/lib/shoryuken/message.rb +4 -13
  21. data/lib/shoryuken/middleware/chain.rb +1 -18
  22. data/lib/shoryuken/middleware/server/auto_delete.rb +3 -8
  23. data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +16 -17
  24. data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +37 -22
  25. data/lib/shoryuken/middleware/server/timing.rb +2 -2
  26. data/lib/shoryuken/options.rb +196 -0
  27. data/lib/shoryuken/polling/base.rb +67 -0
  28. data/lib/shoryuken/polling/strict_priority.rb +77 -0
  29. data/lib/shoryuken/polling/weighted_round_robin.rb +66 -0
  30. data/lib/shoryuken/processor.rb +27 -25
  31. data/lib/shoryuken/queue.rb +28 -9
  32. data/lib/shoryuken/runner.rb +131 -0
  33. data/lib/shoryuken/util.rb +1 -9
  34. data/lib/shoryuken/version.rb +1 -1
  35. data/lib/shoryuken/worker.rb +9 -1
  36. data/lib/shoryuken.rb +47 -157
  37. data/shoryuken.gemspec +7 -7
  38. data/spec/integration/launcher_spec.rb +15 -8
  39. data/spec/shoryuken/client_spec.rb +2 -45
  40. data/spec/shoryuken/default_worker_registry_spec.rb +12 -10
  41. data/spec/shoryuken/environment_loader_spec.rb +54 -0
  42. data/spec/shoryuken/fetcher_spec.rb +27 -46
  43. data/spec/shoryuken/manager_spec.rb +80 -93
  44. data/spec/shoryuken/middleware/chain_spec.rb +0 -24
  45. data/spec/shoryuken/middleware/server/auto_delete_spec.rb +2 -2
  46. data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb +7 -3
  47. data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +56 -33
  48. data/spec/shoryuken/middleware/server/timing_spec.rb +5 -3
  49. data/spec/shoryuken/options_spec.rb +100 -0
  50. data/spec/shoryuken/polling/strict_priority_spec.rb +140 -0
  51. data/spec/shoryuken/polling/weighted_round_robin_spec.rb +99 -0
  52. data/spec/shoryuken/processor_spec.rb +20 -37
  53. data/spec/shoryuken/queue_spec.rb +72 -26
  54. data/spec/shoryuken/{cli_spec.rb → runner_spec.rb} +11 -26
  55. data/spec/shoryuken_spec.rb +1 -48
  56. data/spec/spec_helper.rb +14 -20
  57. data/test_workers/endless_interruptive_worker.rb +41 -0
  58. data/test_workers/endless_uninterruptive_worker.rb +44 -0
  59. metadata +39 -34
  60. data/lib/shoryuken/aws_config.rb +0 -64
  61. data/lib/shoryuken/cli.rb +0 -215
  62. data/lib/shoryuken/sns_arn.rb +0 -27
  63. data/lib/shoryuken/topic.rb +0 -17
  64. data/spec/shoryuken/sns_arn_spec.rb +0 -42
  65. data/spec/shoryuken/topic_spec.rb +0 -32
  66. data/spec/shoryuken_endpoint.yml +0 -6
@@ -1,122 +1,109 @@
1
1
  require 'spec_helper'
2
2
  require 'shoryuken/manager'
3
3
 
4
- RSpec.describe Shoryuken::Manager do
5
- subject do
6
- condvar = double(:condvar)
7
- allow(condvar).to receive(:signal).and_return(nil)
8
- Shoryuken::Manager.new(condvar)
9
- end
10
-
11
- describe 'Invalid concurrency setting' do
12
- it 'raises ArgumentError if concurrency is not positive number' do
13
- Shoryuken.options[:concurrency] = -1
14
- expect { Shoryuken::Manager.new(nil) }
15
- .to raise_error(ArgumentError, 'Concurrency value -1 is invalid, it needs to be a positive number')
16
- end
17
-
4
+ RSpec::Matchers.define :queue_config_of do |expected|
5
+ match do |actual|
6
+ actual.name == expected
18
7
  end
8
+ end
19
9
 
20
- describe 'Auto Scaling' do
21
- it 'decreases weight' do
22
- queue1 = 'shoryuken'
23
- queue2 = 'uppercut'
10
+ RSpec.describe Shoryuken::Manager do
11
+ let(:queue) { 'default' }
12
+ let(:queues) { [queue] }
13
+ let(:polling_strategy) { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
14
+ let(:fetcher) { double Shoryuken::Fetcher }
15
+ let(:concurrency) { 1 }
16
+ let(:executor) { Concurrent::ImmediateExecutor.new }
24
17
 
25
- Shoryuken.queues.clear
26
- # [shoryuken, 2]
27
- # [uppercut, 1]
28
- Shoryuken.queues << queue1
29
- Shoryuken.queues << queue1
30
- Shoryuken.queues << queue2
18
+ subject { Shoryuken::Manager.new(fetcher, polling_strategy, concurrency, executor) }
31
19
 
32
- expect(subject.instance_variable_get('@queues')).to eq [queue1, queue2]
20
+ before do
21
+ allow(fetcher).to receive(:fetch).and_return([])
22
+ end
33
23
 
34
- subject.pause_queue!(queue1)
24
+ after do
25
+ Shoryuken.options[:concurrency] = 1
26
+ TestWorker.get_shoryuken_options['batch'] = false
27
+ end
35
28
 
36
- expect(subject.instance_variable_get('@queues')).to eq [queue2]
29
+ describe '#stop' do
30
+ specify do
31
+ allow(subject).to receive(:running?).and_return(true, true, false)
32
+ expect(subject).to receive(:dispatch).once.and_call_original
33
+ expect(subject).to receive(:dispatch_loop).twice.and_call_original
34
+ subject.start
37
35
  end
36
+ end
38
37
 
39
- it 'increases weight' do
40
- queue1 = 'shoryuken'
41
- queue2 = 'uppercut'
42
-
43
- Shoryuken.queues.clear
44
- # [shoryuken, 3]
45
- # [uppercut, 1]
46
- Shoryuken.queues << queue1
47
- Shoryuken.queues << queue1
48
- Shoryuken.queues << queue1
49
- Shoryuken.queues << queue2
50
-
51
- expect(subject.instance_variable_get('@queues')).to eq [queue1, queue2]
52
- subject.pause_queue!(queue1)
53
- expect(subject.instance_variable_get('@queues')).to eq [queue2]
54
-
55
- subject.rebalance_queue_weight!(queue1)
56
- expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1]
57
-
58
- subject.rebalance_queue_weight!(queue1)
59
- expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1, queue1]
60
-
61
- subject.rebalance_queue_weight!(queue1)
62
- expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1, queue1, queue1]
38
+ describe '#start' do
39
+ before do
40
+ # prevent dispatch loop
41
+ allow(subject).to receive(:running?).and_return(true, true, false)
63
42
  end
64
43
 
65
- it 'adds queue back' do
66
- queue1 = 'shoryuken'
67
- queue2 = 'uppercut'
68
-
69
- Shoryuken.queues.clear
70
- # [shoryuken, 2]
71
- # [uppercut, 1]
72
- Shoryuken.queues << queue1
73
- Shoryuken.queues << queue1
74
- Shoryuken.queues << queue2
75
-
76
- Shoryuken.options[:delay] = 0.1
77
-
78
- fetcher = double('Fetcher').as_null_object
79
- subject.fetcher = fetcher
80
-
81
- subject.pause_queue!(queue1)
82
- expect(subject.instance_variable_get('@queues')).to eq [queue2]
44
+ it 'pauses when there are no active queues' do
45
+ expect(polling_strategy).to receive(:next_queue).and_return(nil)
46
+ expect(subject).to receive(:dispatch).and_call_original
47
+ subject.start
48
+ end
83
49
 
84
- sleep 0.5
50
+ it 'calls dispatch_batch if worker wants batches' do
51
+ TestWorker.get_shoryuken_options['batch'] = true
52
+ expect(subject).to receive(:dispatch_batch).with(queue_config_of(queue))
53
+ subject.start
54
+ end
85
55
 
86
- expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1]
56
+ it 'calls dispatch_single_messages if worker wants single messages' do
57
+ expect(subject).to receive(:dispatch_single_messages).with(queue_config_of(queue))
58
+ subject.start
87
59
  end
88
60
  end
89
61
 
90
- describe '#next_queue' do
91
- it 'returns queues' do
92
- queue1 = 'shoryuken'
93
- queue2 = 'uppercut'
94
-
95
- Shoryuken.queues.clear
62
+ describe '#dispatch' do
63
+ before do
64
+ allow(subject).to receive(:running?).and_return(true, true, false)
65
+ end
96
66
 
97
- Shoryuken.register_worker queue1, TestWorker
98
- Shoryuken.register_worker queue2, TestWorker
67
+ specify do
68
+ message = ['test1']
69
+ messages = [message]
70
+ q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
99
71
 
100
- Shoryuken.queues << queue1
101
- Shoryuken.queues << queue2
72
+ expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
73
+ expect(subject).to receive(:fire_event).with(:dispatch)
74
+ expect(Shoryuken::Processor).to receive(:process).with(q, message)
75
+ expect(Shoryuken.logger).to_not receive(:info)
102
76
 
103
- expect(subject.send :next_queue).to eq queue1
104
- expect(subject.send :next_queue).to eq queue2
77
+ subject.send(:dispatch)
105
78
  end
106
79
 
107
- it 'skips when no worker' do
108
- queue1 = 'shoryuken'
109
- queue2 = 'uppercut'
80
+ context 'when batch' do
81
+ specify do
82
+ messages = %w(test1 test2 test3)
83
+ q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
110
84
 
111
- Shoryuken.queues.clear
85
+ expect(fetcher).to receive(:fetch).with(q, described_class::BATCH_LIMIT).and_return(messages)
86
+ expect(subject).to receive(:fire_event).with(:dispatch)
87
+ allow(subject).to receive(:batched_queue?).with(q).and_return(true)
88
+ expect(Shoryuken::Processor).to receive(:process).with(q, messages)
89
+ expect(Shoryuken.logger).to_not receive(:info)
112
90
 
113
- Shoryuken.register_worker queue2, TestWorker
114
-
115
- Shoryuken.queues << queue1
116
- Shoryuken.queues << queue2
91
+ subject.send(:dispatch)
92
+ end
93
+ end
94
+ end
117
95
 
118
- expect(subject.send :next_queue).to eq queue2
119
- expect(subject.send :next_queue).to eq queue2
96
+ describe '#dispatch_single_messages' do
97
+ let(:concurrency) { 3 }
98
+
99
+ it 'assings messages from batch one by one' do
100
+ q = polling_strategy.next_queue
101
+ messages = [1, 2, 3]
102
+ expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
103
+ expect_any_instance_of(described_class).to receive(:assign).with(q.name, 1)
104
+ expect_any_instance_of(described_class).to receive(:assign).with(q.name, 2)
105
+ expect_any_instance_of(described_class).to receive(:assign).with(q.name, 3)
106
+ subject.send(:dispatch_single_messages, q)
120
107
  end
121
108
  end
122
109
  end
@@ -44,28 +44,4 @@ describe Shoryuken::Middleware::Chain do
44
44
  expect(final_action).to eq nil
45
45
  expect(recorder).to eq []
46
46
  end
47
-
48
- class DeprecatedMiddleware
49
- def call(worker_instance, queue, sqs_msg)
50
- @@success = true
51
- end
52
-
53
- def self.success?
54
- !!@@success
55
- end
56
- end
57
-
58
- it 'patches deprecated middleware' do
59
- subject.clear
60
-
61
- expect(Shoryuken.logger).to receive(:warn) do |&block|
62
- expect(block.call).to eq('[DEPRECATION] DeprecatedMiddleware#call(worker_instance, queue, sqs_msg) is deprecated. Please use DeprecatedMiddleware#call(worker_instance, queue, sqs_msg, body)')
63
- end
64
-
65
- subject.add DeprecatedMiddleware
66
-
67
- subject.invoke TestWorker, 'test', double('SQS msg', body: 'test'), 'test'
68
-
69
- expect(DeprecatedMiddleware.success?).to eq true
70
- end
71
47
  end
@@ -55,8 +55,8 @@ describe Shoryuken::Middleware::Server::AutoDelete do
55
55
  expect(sqs_queue).to_not receive(:delete_messages)
56
56
 
57
57
  expect {
58
- subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'Error' }
59
- }.to raise_error(RuntimeError, 'Error')
58
+ subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' }
59
+ }.to raise_error('failed')
60
60
  end
61
61
  end
62
62
  end
@@ -1,6 +1,6 @@
1
1
  require 'spec_helper'
2
2
 
3
- describe Shoryuken::Middleware::Server::AutoExtendVisibility do
3
+ RSpec.describe Shoryuken::Middleware::Server::AutoExtendVisibility do
4
4
  let(:queue) { 'default' }
5
5
  let(:visibility_timeout) { 3 }
6
6
  let(:extend_upfront) { 1 }
@@ -12,8 +12,6 @@ describe Shoryuken::Middleware::Server::AutoExtendVisibility do
12
12
 
13
13
  # We need to run our worker inside actor context.
14
14
  class Runner
15
- include Celluloid
16
-
17
15
  def run_and_sleep(worker, queue, sqs_msg, interval)
18
16
  Shoryuken::Middleware::Server::AutoExtendVisibility.new.call(worker, queue, sqs_msg, sqs_msg.body) do
19
17
  sleep interval
@@ -34,6 +32,12 @@ describe Shoryuken::Middleware::Server::AutoExtendVisibility do
34
32
  stub_const('Shoryuken::Middleware::Server::AutoExtendVisibility::EXTEND_UPFRONT_SECONDS', extend_upfront)
35
33
  end
36
34
 
35
+ context 'when batch worker' do
36
+ it 'yields' do
37
+ expect { |b| subject.call(TestWorker.new, nil, [], nil, &b) }.to yield_control
38
+ end
39
+ end
40
+
37
41
  it 'extends message visibility if jobs takes a long time' do
38
42
  TestWorker.get_shoryuken_options['auto_visibility_timeout'] = true
39
43
 
@@ -1,6 +1,7 @@
1
1
  require 'spec_helper'
2
2
 
3
- describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
3
+ # rubocop:disable Metrics/BlockLength, Metrics/BlockDelimiters
4
+ RSpec.describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
4
5
  let(:queue) { 'default' }
5
6
  let(:sqs_queue) { double Shoryuken::Queue }
6
7
  let(:sqs_msg) { double Shoryuken::Message, queue_url: queue, body: 'test', receipt_handle: SecureRandom.uuid,
@@ -10,8 +11,14 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
10
11
  allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
11
12
  end
12
13
 
13
- context 'when a job succeeds' do
14
- it 'does not retry the job' do
14
+ context 'when batch worker' do
15
+ it 'yields' do
16
+ expect { |b| subject.call(TestWorker.new, nil, [], nil, &b) }.to yield_control
17
+ end
18
+ end
19
+
20
+ context 'when no exception' do
21
+ it 'does not retry' do
15
22
  TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
16
23
 
17
24
  expect(sqs_msg).not_to receive(:change_visibility)
@@ -20,62 +27,78 @@ describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
20
27
  end
21
28
  end
22
29
 
23
- context 'when a job throws an exception' do
30
+ context 'when an error' do
31
+ context "and retry_intervals isn't set" do
32
+ it 'does not retry' do
33
+ expect(sqs_msg).not_to receive(:change_visibility)
24
34
 
25
- it 'does not retry the job by default' do
26
- expect(sqs_msg).not_to receive(:change_visibility)
35
+ expect {
36
+ subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'Error' }
37
+ }.to raise_error(RuntimeError, 'Error')
38
+ end
39
+ end
27
40
 
28
- expect {
29
- subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'Error' }
30
- }.to raise_error(RuntimeError, 'Error')
41
+ context 'and retry_intervals is a lambda' do
42
+ it 'retries' do
43
+ TestWorker.get_shoryuken_options['retry_intervals'] = ->(_attempts) { 500 }
44
+
45
+ allow(sqs_msg).to receive(:queue) { sqs_queue }
46
+ expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 500)
47
+
48
+ expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
49
+ end
31
50
  end
32
51
 
33
- it 'does not retry the job if :retry_intervals is empty' do
34
- TestWorker.get_shoryuken_options['retry_intervals'] = []
52
+ context 'and retry_intervals is empty' do
53
+ it 'does not retry' do
54
+ TestWorker.get_shoryuken_options['retry_intervals'] = []
35
55
 
36
- expect(sqs_msg).not_to receive(:change_visibility)
56
+ expect(sqs_msg).not_to receive(:change_visibility)
37
57
 
38
- expect {
39
- subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'Error' }
40
- }.to raise_error(RuntimeError, 'Error')
58
+ expect {
59
+ subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'Error' }
60
+ }.to raise_error(RuntimeError, 'Error')
61
+ end
41
62
  end
42
63
 
43
- it 'retries the job if :retry_intervals is non-empty' do
64
+ it 'uses first interval ' do
44
65
  TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
45
66
 
46
- allow(sqs_msg).to receive(:queue){ sqs_queue }
67
+ allow(sqs_msg).to receive(:queue) { sqs_queue }
47
68
  expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 300)
48
69
 
49
- expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
70
+ expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
50
71
  end
51
72
 
52
- it 'retries the job with exponential backoff' do
73
+ it 'uses matching interval' do
53
74
  TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
54
75
 
55
- allow(sqs_msg).to receive(:attributes){ {'ApproximateReceiveCount' => 2 } }
56
- allow(sqs_msg).to receive(:queue){ sqs_queue }
76
+ allow(sqs_msg).to receive(:attributes) { { 'ApproximateReceiveCount' => 2 } }
77
+ allow(sqs_msg).to receive(:queue) { sqs_queue }
57
78
  expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
58
79
 
59
- expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
80
+ expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
60
81
  end
61
82
 
62
- it 'uses the last retry interval when :receive_count exceeds the size of :retry_intervals' do
63
- TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
83
+ context 'when attempts exceeds retry_intervals' do
84
+ it 'uses last interval' do
85
+ TestWorker.get_shoryuken_options['retry_intervals'] = [300, 1800]
64
86
 
65
- allow(sqs_msg).to receive(:attributes){ {'ApproximateReceiveCount' => 3 } }
66
- allow(sqs_msg).to receive(:queue){ sqs_queue }
67
- expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
87
+ allow(sqs_msg).to receive(:attributes) { { 'ApproximateReceiveCount' => 3 } }
88
+ allow(sqs_msg).to receive(:queue) { sqs_queue }
89
+ expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 1800)
68
90
 
69
- expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
91
+ expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
92
+ end
70
93
  end
71
94
 
72
- it 'limits the visibility timeout to 12 hours from receipt of message' do
73
- TestWorker.get_shoryuken_options['retry_intervals'] = [86400]
95
+ it 'limits the visibility timeout to 12 hours' do
96
+ TestWorker.get_shoryuken_options['retry_intervals'] = [86_400]
74
97
 
75
- allow(sqs_msg).to receive(:queue){ sqs_queue }
76
- expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 43198)
98
+ allow(sqs_msg).to receive(:queue) { sqs_queue }
99
+ expect(sqs_msg).to receive(:change_visibility).with(visibility_timeout: 43_198)
77
100
 
78
- expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise } }.not_to raise_error
101
+ expect { subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) { raise 'failed' } }.not_to raise_error
79
102
  end
80
103
  end
81
104
  end
@@ -1,14 +1,16 @@
1
1
  require 'spec_helper'
2
2
 
3
- describe Shoryuken::Middleware::Server::Timing do
3
+ RSpec.describe Shoryuken::Middleware::Server::Timing do
4
4
  let(:queue) { 'default' }
5
5
  let(:sqs_queue) { double Shoryuken::Queue, visibility_timeout: 60 }
6
6
 
7
7
  let(:sqs_msg) do
8
- double Shoryuken::Message,
8
+ double(
9
+ Shoryuken::Message,
9
10
  queue_url: queue,
10
11
  body: 'test',
11
12
  message_id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e'
13
+ )
12
14
  end
13
15
 
14
16
  before do
@@ -28,7 +30,7 @@ describe Shoryuken::Middleware::Server::Timing do
28
30
 
29
31
  context 'when exceeded the `visibility_timeout`' do
30
32
  it 'logs exceeded' do
31
- allow(subject).to receive(:elapsed).and_return(120000)
33
+ allow(subject).to receive(:elapsed).and_return(120_000)
32
34
 
33
35
  expect(Shoryuken.logger).to receive(:info) do |&block|
34
36
  expect(block.call).to match(/started at/)
@@ -0,0 +1,100 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe Shoryuken::Options do
4
+ describe '.add_group' do
5
+ before do
6
+ Shoryuken.groups.clear
7
+ Shoryuken.add_group('group1', 25)
8
+ Shoryuken.add_group('group2', 25)
9
+ end
10
+
11
+ specify do
12
+ described_class.add_queue('queue1', 1, 'group1')
13
+ described_class.add_queue('queue2', 2, 'group2')
14
+
15
+ expect(described_class.groups['group1'][:queues]).to eq(%w(queue1))
16
+ expect(described_class.groups['group2'][:queues]).to eq(%w(queue2 queue2))
17
+ end
18
+ end
19
+
20
+ describe '.ungrouped_queues' do
21
+ before do
22
+ Shoryuken.groups.clear
23
+ Shoryuken.add_group('group1', 25)
24
+ Shoryuken.add_group('group2', 25)
25
+ end
26
+
27
+ specify do
28
+ described_class.add_queue('queue1', 1, 'group1')
29
+ described_class.add_queue('queue2', 2, 'group2')
30
+
31
+ expect(described_class.ungrouped_queues).to eq(%w(queue1 queue2 queue2))
32
+ end
33
+ end
34
+
35
+ describe '.sqs_client_receive_message_opts' do
36
+ before do
37
+ Shoryuken.sqs_client_receive_message_opts
38
+ end
39
+
40
+ specify do
41
+ Shoryuken.sqs_client_receive_message_opts = { test: 1 }
42
+ expect(Shoryuken.sqs_client_receive_message_opts).to eq('default' => { test: 1 })
43
+
44
+ Shoryuken.sqs_client_receive_message_opts['group1'] = { test: 2 }
45
+
46
+ expect(Shoryuken.sqs_client_receive_message_opts).to eq(
47
+ 'default' => { test: 1 },
48
+ 'group1' => { test: 2 },
49
+ )
50
+ end
51
+ end
52
+
53
+ describe '.register_worker' do
54
+ it 'registers a worker' do
55
+ described_class.worker_registry.clear
56
+ described_class.register_worker('default', TestWorker)
57
+ expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
58
+ end
59
+
60
+ it 'registers a batchable worker' do
61
+ described_class.worker_registry.clear
62
+ TestWorker.get_shoryuken_options['batch'] = true
63
+ described_class.register_worker('default', TestWorker)
64
+ expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
65
+ end
66
+
67
+ it 'allows multiple workers' do
68
+ described_class.worker_registry.clear
69
+ described_class.register_worker('default', TestWorker)
70
+ expect(described_class.worker_registry.workers('default')).to eq([TestWorker])
71
+
72
+ class Test2Worker
73
+ include Shoryuken::Worker
74
+
75
+ shoryuken_options queue: 'default'
76
+
77
+ def perform(sqs_msg, body); end
78
+ end
79
+
80
+ expect(described_class.worker_registry.workers('default')).to eq([Test2Worker])
81
+ end
82
+
83
+ it 'raises an exception when mixing batchable with non batchable' do
84
+ described_class.worker_registry.clear
85
+ TestWorker.get_shoryuken_options['batch'] = true
86
+ described_class.register_worker('default', TestWorker)
87
+
88
+ expect {
89
+ class BatchableWorker
90
+ include Shoryuken::Worker
91
+
92
+ shoryuken_options queue: 'default', batch: true
93
+
94
+ def perform(sqs_msg, body); end
95
+ end
96
+ }.to raise_error("Could not register BatchableWorker for default, because TestWorker is already registered for this queue, " \
97
+ "and Shoryuken doesn't support a batchable worker for a queue with multiple workers")
98
+ end
99
+ end
100
+ end
@@ -0,0 +1,140 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe Shoryuken::Polling::StrictPriority do
4
+ let(:queue1) { 'shoryuken' }
5
+ let(:queue2) { 'uppercut' }
6
+ let(:queue3) { 'other' }
7
+ let(:queues) { Array.new }
8
+ subject { Shoryuken::Polling::StrictPriority.new(queues) }
9
+
10
+ describe '#next_queue' do
11
+ it 'cycles when declared desc' do
12
+ # [shoryuken, 2]
13
+ # [uppercut, 1]
14
+ queues << queue1
15
+ queues << queue1
16
+ queues << queue2
17
+
18
+ expect(subject.next_queue).to eq(queue1)
19
+ expect(subject.next_queue).to eq(queue2)
20
+ expect(subject.next_queue).to eq(queue1)
21
+ expect(subject.next_queue).to eq(queue2)
22
+ end
23
+
24
+ it 'cycles when declared asc' do
25
+ # [uppercut, 1]
26
+ # [shoryuken, 2]
27
+ queues << queue2
28
+ queues << queue1
29
+ queues << queue1
30
+
31
+ expect(subject.next_queue).to eq(queue1)
32
+ expect(subject.next_queue).to eq(queue2)
33
+ expect(subject.next_queue).to eq(queue1)
34
+ expect(subject.next_queue).to eq(queue2)
35
+ end
36
+
37
+ it 'returns nil if there are no active queues' do
38
+ expect(subject.next_queue).to eq(nil)
39
+ end
40
+
41
+ it 'unpauses queues whose pause is expired' do
42
+ # [shoryuken, 3]
43
+ # [uppercut, 2]
44
+ # [other, 1]
45
+ queues << queue1
46
+ queues << queue1
47
+ queues << queue1
48
+ queues << queue2
49
+ queues << queue2
50
+ queues << queue3
51
+
52
+ allow(subject).to receive(:delay).and_return(10)
53
+
54
+ now = Time.now
55
+ allow(Time).to receive(:now).and_return(now)
56
+
57
+ # pause the second queue, see it loop between 1 and 3
58
+ subject.messages_found(queue2, 0)
59
+ expect(subject.next_queue).to eq(queue1)
60
+ expect(subject.next_queue).to eq(queue3)
61
+ expect(subject.next_queue).to eq(queue1)
62
+
63
+ now += 5
64
+ allow(Time).to receive(:now).and_return(now)
65
+
66
+ # pause the first queue, see it repeat 3
67
+ subject.messages_found(queue1, 0)
68
+ expect(subject.next_queue).to eq(queue3)
69
+ expect(subject.next_queue).to eq(queue3)
70
+
71
+ # pause the third queue, see it have nothing
72
+ subject.messages_found(queue3, 0)
73
+ expect(subject.next_queue).to eq(nil)
74
+
75
+ # unpause queue 2
76
+ now += 6
77
+ allow(Time).to receive(:now).and_return(now)
78
+ expect(subject.next_queue).to eq(queue2)
79
+
80
+ # unpause queues 1 and 3
81
+ now += 6
82
+ allow(Time).to receive(:now).and_return(now)
83
+ expect(subject.next_queue).to eq(queue1)
84
+ expect(subject.next_queue).to eq(queue2)
85
+ expect(subject.next_queue).to eq(queue3)
86
+ end
87
+ end
88
+
89
+ describe '#messages_found' do
90
+ it 'pauses a queue if there are no messages found' do
91
+ # [shoryuken, 2]
92
+ # [uppercut, 1]
93
+ queues << queue1
94
+ queues << queue1
95
+ queues << queue2
96
+
97
+ expect(subject.active_queues).to eq([[queue1, 2], [queue2, 1]])
98
+ expect(subject).to receive(:pause).with(queue1).and_call_original
99
+ subject.messages_found(queue1, 0)
100
+ expect(subject.active_queues).to eq([[queue2, 1]])
101
+ end
102
+
103
+ it 'continues to queue the highest priority queue if messages are found' do
104
+ # [shoryuken, 3]
105
+ # [uppercut, 2]
106
+ # [other, 1]
107
+ queues << queue1
108
+ queues << queue1
109
+ queues << queue1
110
+ queues << queue2
111
+ queues << queue2
112
+ queues << queue3
113
+
114
+ expect(subject.next_queue).to eq(queue1)
115
+ subject.messages_found(queue1, 1)
116
+ expect(subject.next_queue).to eq(queue1)
117
+ subject.messages_found(queue1, 1)
118
+ expect(subject.next_queue).to eq(queue1)
119
+ end
120
+
121
+ it 'resets the priorities if messages are found part way' do
122
+ # [shoryuken, 3]
123
+ # [uppercut, 2]
124
+ # [other, 1]
125
+ queues << queue1
126
+ queues << queue1
127
+ queues << queue1
128
+ queues << queue2
129
+ queues << queue2
130
+ queues << queue3
131
+
132
+ expect(subject.next_queue).to eq(queue1)
133
+ expect(subject.next_queue).to eq(queue2)
134
+ subject.messages_found(queue2, 1)
135
+ expect(subject.next_queue).to eq(queue1)
136
+ expect(subject.next_queue).to eq(queue2)
137
+ expect(subject.next_queue).to eq(queue3)
138
+ end
139
+ end
140
+ end