shoryuken 5.0.4 → 5.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/specs.yml +62 -0
  3. data/.reek.yml +5 -0
  4. data/.rubocop.yml +1 -1
  5. data/Appraisals +36 -0
  6. data/CHANGELOG.md +63 -0
  7. data/Gemfile +3 -1
  8. data/README.md +24 -2
  9. data/Rakefile +15 -1
  10. data/bin/cli/sqs.rb +50 -5
  11. data/gemfiles/.gitignore +1 -0
  12. data/gemfiles/aws_sdk_core_2.gemfile +21 -0
  13. data/gemfiles/rails_4_2.gemfile +20 -0
  14. data/gemfiles/rails_5_2.gemfile +21 -0
  15. data/gemfiles/rails_6_0.gemfile +21 -0
  16. data/gemfiles/rails_6_1.gemfile +21 -0
  17. data/lib/shoryuken/environment_loader.rb +7 -1
  18. data/lib/shoryuken/extensions/active_job_adapter.rb +25 -18
  19. data/lib/shoryuken/extensions/active_job_extensions.rb +38 -0
  20. data/lib/shoryuken/launcher.rb +1 -0
  21. data/lib/shoryuken/manager.rb +24 -5
  22. data/lib/shoryuken/options.rb +1 -0
  23. data/lib/shoryuken/polling/base.rb +2 -0
  24. data/lib/shoryuken/polling/strict_priority.rb +6 -0
  25. data/lib/shoryuken/polling/weighted_round_robin.rb +11 -0
  26. data/lib/shoryuken/queue.rb +39 -11
  27. data/lib/shoryuken/version.rb +1 -1
  28. data/lib/shoryuken.rb +1 -0
  29. data/shoryuken.gemspec +0 -1
  30. data/spec/integration/launcher_spec.rb +29 -2
  31. data/spec/shared_examples_for_active_job.rb +226 -9
  32. data/spec/shoryuken/environment_loader_spec.rb +22 -2
  33. data/spec/shoryuken/extensions/active_job_adapter_spec.rb +1 -1
  34. data/spec/shoryuken/extensions/active_job_base_spec.rb +84 -0
  35. data/spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb +4 -0
  36. data/spec/shoryuken/extensions/active_job_wrapper_spec.rb +20 -0
  37. data/spec/shoryuken/manager_spec.rb +35 -1
  38. data/spec/shoryuken/polling/strict_priority_spec.rb +10 -0
  39. data/spec/shoryuken/polling/weighted_round_robin_spec.rb +10 -0
  40. data/spec/shoryuken/queue_spec.rb +23 -0
  41. data/spec/spec_helper.rb +5 -9
  42. metadata +20 -22
  43. data/.travis.yml +0 -34
  44. data/Gemfile.aws-sdk-core-v2 +0 -13
@@ -1,30 +1,44 @@
1
+ require 'active_job'
2
+ require 'shoryuken/extensions/active_job_extensions'
3
+
4
+ # Stand-in for a job class specified by the user
5
+ class TestJob < ActiveJob::Base; end
6
+
1
7
  # rubocop:disable Metrics/BlockLength
2
8
  RSpec.shared_examples 'active_job_adapters' do
3
- let(:job) { double 'Job', id: '123', queue_name: 'queue' }
9
+ let(:job_sqs_send_message_parameters) { {} }
10
+ let(:job) do
11
+ job = TestJob.new
12
+ job.sqs_send_message_parameters = job_sqs_send_message_parameters
13
+ job
14
+ end
4
15
  let(:fifo) { false }
5
16
  let(:queue) { double 'Queue', fifo?: fifo }
6
17
 
7
18
  before do
8
19
  allow(Shoryuken::Client).to receive(:queues).with(job.queue_name).and_return(queue)
9
- allow(job).to receive(:serialize).and_return(
10
- 'job_class' => 'Worker',
11
- 'job_id' => job.id,
12
- 'queue_name' => job.queue_name,
13
- 'arguments' => nil,
14
- 'locale' => nil
15
- )
16
20
  end
17
21
 
18
22
  describe '#enqueue' do
19
23
  specify do
20
24
  expect(queue).to receive(:send_message) do |hash|
21
25
  expect(hash[:message_deduplication_id]).to_not be
26
+ expect(hash[:message_attributes]['shoryuken_class'][:string_value]).to eq(described_class::JobWrapper.to_s)
27
+ expect(hash[:message_attributes]['shoryuken_class'][:data_type]).to eq("String")
28
+ expect(hash[:message_attributes].keys).to eq(['shoryuken_class'])
22
29
  end
23
30
  expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
24
31
 
25
32
  subject.enqueue(job)
26
33
  end
27
34
 
35
+ it "should mutate the job's sqs_send_message_parameters reference to match those sent to the queue" do
36
+ expect(queue).to receive(:send_message) do |options|
37
+ expect(options).to be(job.sqs_send_message_parameters)
38
+ end
39
+ subject.enqueue(job)
40
+ end
41
+
28
42
  context 'when fifo' do
29
43
  let(:fifo) { true }
30
44
 
@@ -38,6 +52,209 @@ RSpec.shared_examples 'active_job_adapters' do
38
52
 
39
53
  subject.enqueue(job)
40
54
  end
55
+
56
+ context 'with message_deduplication_id' do
57
+ context 'when message_deduplication_id is specified in options' do
58
+ it 'should enqueue a message with the deduplication_id specified in options' do
59
+ expect(queue).to receive(:send_message) do |hash|
60
+ expect(hash[:message_deduplication_id]).to eq('options-dedupe-id')
61
+ end
62
+ subject.enqueue(job, message_deduplication_id: 'options-dedupe-id')
63
+ end
64
+ end
65
+
66
+ context 'when message_deduplication_id is specified on the job' do
67
+ let(:job_sqs_send_message_parameters) { { message_deduplication_id: 'job-dedupe-id' } }
68
+
69
+ it 'should enqueue a message with the deduplication_id specified on the job' do
70
+ expect(queue).to receive(:send_message) do |hash|
71
+ expect(hash[:message_deduplication_id]).to eq('job-dedupe-id')
72
+ end
73
+ subject.enqueue job
74
+ end
75
+ end
76
+
77
+ context 'when message_deduplication_id is specified on the job and also in options' do
78
+ let(:job_sqs_send_message_parameters) { { message_deduplication_id: 'job-dedupe-id' } }
79
+
80
+ it 'should enqueue a message with the deduplication_id specified in options' do
81
+ expect(queue).to receive(:send_message) do |hash|
82
+ expect(hash[:message_deduplication_id]).to eq('options-dedupe-id')
83
+ end
84
+ subject.enqueue(job, message_deduplication_id: 'options-dedupe-id')
85
+ end
86
+ end
87
+ end
88
+ end
89
+
90
+ context 'with message_group_id' do
91
+ context 'when message_group_id is specified in options' do
92
+ it 'should enqueue a message with the group_id specified in options' do
93
+ expect(queue).to receive(:send_message) do |hash|
94
+ expect(hash[:message_group_id]).to eq('options-group-id')
95
+ end
96
+ subject.enqueue(job, message_group_id: 'options-group-id')
97
+ end
98
+ end
99
+
100
+ context 'when message_group_id is specified on the job' do
101
+ let(:job_sqs_send_message_parameters) { { message_group_id: 'job-group-id' } }
102
+
103
+ it 'should enqueue a message with the group_id specified on the job' do
104
+ expect(queue).to receive(:send_message) do |hash|
105
+ expect(hash[:message_group_id]).to eq('job-group-id')
106
+ end
107
+ subject.enqueue job
108
+ end
109
+ end
110
+
111
+ context 'when message_group_id is specified on the job and also in options' do
112
+ let(:job_sqs_send_message_parameters) { { message_group_id: 'job-group-id' } }
113
+
114
+ it 'should enqueue a message with the group_id specified in options' do
115
+ expect(queue).to receive(:send_message) do |hash|
116
+ expect(hash[:message_group_id]).to eq('options-group-id')
117
+ end
118
+ subject.enqueue(job, message_group_id: 'options-group-id')
119
+ end
120
+ end
121
+ end
122
+
123
+ context 'with additional message attributes' do
124
+ it 'should combine with activejob attributes' do
125
+ custom_message_attributes = {
126
+ 'tracer_id' => {
127
+ string_value: SecureRandom.hex,
128
+ data_type: 'String'
129
+ }
130
+ }
131
+
132
+ expect(queue).to receive(:send_message) do |hash|
133
+ expect(hash[:message_attributes]['shoryuken_class'][:string_value]).to eq(described_class::JobWrapper.to_s)
134
+ expect(hash[:message_attributes]['shoryuken_class'][:data_type]).to eq("String")
135
+ expect(hash[:message_attributes]['tracer_id'][:string_value]).to eq(custom_message_attributes['tracer_id'][:string_value])
136
+ expect(hash[:message_attributes]['tracer_id'][:data_type]).to eq("String")
137
+ end
138
+ expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
139
+
140
+ subject.enqueue(job, message_attributes: custom_message_attributes)
141
+ end
142
+
143
+ context 'when message_attributes are specified on the job' do
144
+ let(:job_sqs_send_message_parameters) do
145
+ {
146
+ message_attributes: {
147
+ 'tracer_id' => {
148
+ data_type: 'String',
149
+ string_value: 'job-value'
150
+ }
151
+ }
152
+ }
153
+ end
154
+
155
+ it 'should enqueue a message with the message_attributes specified on the job' do
156
+ expect(queue).to receive(:send_message) do |hash|
157
+ expect(hash[:message_attributes]['tracer_id']).to eq({ data_type: 'String', string_value: 'job-value' })
158
+ expect(hash[:message_attributes]['shoryuken_class']).to eq({ data_type: 'String', string_value: described_class::JobWrapper.to_s })
159
+ end
160
+ subject.enqueue job
161
+ end
162
+ end
163
+
164
+ context 'when message_attributes are specified on the job and also in options' do
165
+ let(:job_sqs_send_message_parameters) do
166
+ {
167
+ message_attributes: {
168
+ 'tracer_id' => {
169
+ data_type: 'String',
170
+ string_value: 'job-value'
171
+ }
172
+ }
173
+ }
174
+ end
175
+
176
+ it 'should enqueue a message with the message_attributes speficied in options' do
177
+ custom_message_attributes = {
178
+ 'options_tracer_id' => {
179
+ string_value: 'options-value',
180
+ data_type: 'String'
181
+ }
182
+ }
183
+
184
+ expect(queue).to receive(:send_message) do |hash|
185
+ expect(hash[:message_attributes]['tracer_id']).to be_nil
186
+ expect(hash[:message_attributes]['options_tracer_id']).to eq({ data_type: 'String', string_value: 'options-value' })
187
+ expect(hash[:message_attributes]['shoryuken_class']).to eq({ data_type: 'String', string_value: described_class::JobWrapper.to_s })
188
+ end
189
+ subject.enqueue job, message_attributes: custom_message_attributes
190
+ end
191
+ end
192
+ end
193
+ end
194
+
195
+ context 'with message_system_attributes' do
196
+ context 'when message_system_attributes are specified in options' do
197
+ it 'should enqueue a message with message_system_attributes specified in options' do
198
+ system_attributes = {
199
+ 'AWSTraceHeader' => {
200
+ string_value: 'trace_id',
201
+ data_type: 'String'
202
+ }
203
+ }
204
+ expect(queue).to receive(:send_message) do |hash|
205
+ expect(hash[:message_system_attributes]['AWSTraceHeader'][:string_value]).to eq('trace_id')
206
+ expect(hash[:message_system_attributes]['AWSTraceHeader'][:data_type]).to eq('String')
207
+ end
208
+ subject.enqueue(job, message_system_attributes: system_attributes)
209
+ end
210
+ end
211
+
212
+ context 'when message_system_attributes are specified on the job' do
213
+ let(:job_sqs_send_message_parameters) do
214
+ {
215
+ message_system_attributes: {
216
+ 'AWSTraceHeader' => {
217
+ string_value: 'job-value',
218
+ data_type: 'String'
219
+ }
220
+ }
221
+ }
222
+ end
223
+
224
+ it 'should enqueue a message with the message_system_attributes specified on the job' do
225
+ expect(queue).to receive(:send_message) do |hash|
226
+ expect(hash[:message_system_attributes]['AWSTraceHeader']).to eq({ data_type: 'String', string_value: 'job-value' })
227
+ end
228
+ subject.enqueue job
229
+ end
230
+ end
231
+
232
+ context 'when message_system_attributes are specified on the job and also in options' do
233
+ let(:job_sqs_send_message_parameters) do
234
+ {
235
+ message_system_attributes: {
236
+ 'job_trace_header' => {
237
+ string_value: 'job-value',
238
+ data_type: 'String'
239
+ }
240
+ }
241
+ }
242
+ end
243
+
244
+ it 'should enqueue a message with the message_system_attributes speficied in options' do
245
+ custom_message_attributes = {
246
+ 'options_trace_header' => {
247
+ string_value: 'options-value',
248
+ data_type: 'String'
249
+ }
250
+ }
251
+
252
+ expect(queue).to receive(:send_message) do |hash|
253
+ expect(hash[:message_system_attributes]['job_trace_header']).to be_nil
254
+ expect(hash[:message_system_attributes]['options_trace_header']).to eq({ data_type: 'String', string_value: 'options-value' })
255
+ end
256
+ subject.enqueue job, message_system_attributes: custom_message_attributes
257
+ end
41
258
  end
42
259
  end
43
260
 
@@ -59,4 +276,4 @@ RSpec.shared_examples 'active_job_adapters' do
59
276
  end
60
277
  end
61
278
  end
62
- # rubocop:enable Metrics/BlockLength
279
+ # rubocop:enable Metrics/BlockLength
@@ -57,11 +57,11 @@ RSpec.describe Shoryuken::EnvironmentLoader do
57
57
  ActiveJob::Base.queue_name_delimiter = '_'
58
58
 
59
59
  allow(Shoryuken).to receive(:active_job?).and_return(true)
60
- end
61
60
 
62
- specify do
63
61
  Shoryuken.active_job_queue_name_prefixing = true
62
+ end
64
63
 
64
+ specify do
65
65
  Shoryuken.options[:queues] = ['queue1', ['queue2', 2]]
66
66
 
67
67
  Shoryuken.options[:groups] = {
@@ -73,6 +73,26 @@ RSpec.describe Shoryuken::EnvironmentLoader do
73
73
  expect(Shoryuken.groups['default'][:queues]).to eq(%w[test_queue1 test_queue2 test_queue2])
74
74
  expect(Shoryuken.groups['group1'][:queues]).to eq(%w[test_group1_queue1 test_group1_queue2])
75
75
  end
76
+
77
+ it 'does not prefix url-based queues' do
78
+ Shoryuken.options[:queues] = ['https://example.com/test_queue1']
79
+ Shoryuken.options[:groups] = {'group1' => {queues: ['https://example.com/test_group1_queue1']}}
80
+
81
+ subject.load
82
+
83
+ expect(Shoryuken.groups['default'][:queues]).to(eq(['https://example.com/test_queue1']))
84
+ expect(Shoryuken.groups['group1'][:queues]).to(eq(['https://example.com/test_group1_queue1']))
85
+ end
86
+
87
+ it 'does not prefix arn-based queues' do
88
+ Shoryuken.options[:queues] = ['arn:aws:sqs:fake-region-1:1234:test_queue1']
89
+ Shoryuken.options[:groups] = {'group1' => {queues: ['arn:aws:sqs:fake-region-1:1234:test_group1_queue1']}}
90
+
91
+ subject.load
92
+
93
+ expect(Shoryuken.groups['default'][:queues]).to(eq(['arn:aws:sqs:fake-region-1:1234:test_queue1']))
94
+ expect(Shoryuken.groups['group1'][:queues]).to(eq(['arn:aws:sqs:fake-region-1:1234:test_group1_queue1']))
95
+ end
76
96
  end
77
97
  describe "#setup_options" do
78
98
  let (:cli_queues) { { "queue1"=> 10, "queue2" => 20 } }
@@ -1,6 +1,6 @@
1
1
  require 'spec_helper'
2
- require 'shoryuken/extensions/active_job_adapter'
3
2
  require 'shared_examples_for_active_job'
3
+ require 'shoryuken/extensions/active_job_adapter'
4
4
 
5
5
  RSpec.describe ActiveJob::QueueAdapters::ShoryukenAdapter do
6
6
  include_examples 'active_job_adapters'
@@ -0,0 +1,84 @@
1
+ require 'spec_helper'
2
+ require 'active_job'
3
+ require 'shoryuken/extensions/active_job_extensions'
4
+ require 'shoryuken/extensions/active_job_adapter'
5
+
6
+ RSpec.describe ActiveJob::Base do
7
+ let(:queue_adapter) { ActiveJob::QueueAdapters::ShoryukenAdapter.new }
8
+
9
+ subject do
10
+ worker_class = Class.new(described_class)
11
+ Object.const_set :MyWorker, worker_class
12
+ worker_class.queue_adapter = queue_adapter
13
+ worker_class
14
+ end
15
+
16
+ after do
17
+ Object.send :remove_const, :MyWorker
18
+ end
19
+
20
+ describe '#perform_now' do
21
+ it 'allows keyward args' do
22
+ collaborator = double 'worker collaborator'
23
+ subject.send(:define_method, :perform) do |**kwargs|
24
+ collaborator.foo(**kwargs)
25
+ end
26
+ expect(collaborator).to receive(:foo).with(foo: 'bar')
27
+ subject.perform_now foo: 'bar'
28
+ end
29
+ end
30
+
31
+ describe '#perform_later' do
32
+ it 'calls enqueue on the adapter with the expected job' do
33
+ expect(queue_adapter).to receive(:enqueue) do |job|
34
+ expect(job.arguments).to eq([1, 2])
35
+ end
36
+
37
+ subject.perform_later 1, 2
38
+ end
39
+
40
+ it 'passes message_group_id to the queue_adapter' do
41
+ expect(queue_adapter).to receive(:enqueue) do |job|
42
+ expect(job.sqs_send_message_parameters[:message_group_id]).to eq('group-2')
43
+ end
44
+
45
+ subject.set(message_group_id: 'group-2').perform_later 1, 2
46
+ end
47
+
48
+ it 'passes message_deduplication_id to the queue_adapter' do
49
+ expect(queue_adapter).to receive(:enqueue) do |job|
50
+ expect(job.sqs_send_message_parameters[:message_deduplication_id]).to eq('dedupe-id')
51
+ end
52
+
53
+ subject.set(message_deduplication_id: 'dedupe-id').perform_later 1, 2
54
+ end
55
+
56
+ it 'passes message_attributes to the queue_adapter' do
57
+ message_attributes = {
58
+ 'custom_tracing_id' => {
59
+ string_value: 'value',
60
+ data_type: 'String'
61
+ }
62
+ }
63
+ expect(queue_adapter).to receive(:enqueue) do |job|
64
+ expect(job.sqs_send_message_parameters[:message_attributes]).to eq(message_attributes)
65
+ end
66
+
67
+ subject.set(message_attributes: message_attributes).perform_later 1, 2
68
+ end
69
+
70
+ it 'passes message_system_attributes to the queue_adapter' do
71
+ message_system_attributes = {
72
+ 'AWSTraceHeader' => {
73
+ string_value: 'trace_id',
74
+ data_type: 'String'
75
+ }
76
+ }
77
+ expect(queue_adapter).to receive(:enqueue) do |job|
78
+ expect(job.sqs_send_message_parameters[:message_system_attributes]).to eq(message_system_attributes)
79
+ end
80
+
81
+ subject.set(message_system_attributes: message_system_attributes).perform_later 1, 2
82
+ end
83
+ end
84
+ end
@@ -10,6 +10,10 @@ RSpec.describe ActiveJob::QueueAdapters::ShoryukenConcurrentSendAdapter do
10
10
  let(:error_handler) { -> {} }
11
11
  let(:success_handler) { -> {} }
12
12
 
13
+ before do
14
+ allow(Concurrent).to receive(:global_io_executor).and_return(Concurrent::ImmediateExecutor.new)
15
+ end
16
+
13
17
  subject { described_class.new(success_handler, error_handler) }
14
18
 
15
19
  context 'when success' do
@@ -0,0 +1,20 @@
1
+ require 'spec_helper'
2
+ require 'active_job'
3
+ require 'shoryuken/extensions/active_job_extensions'
4
+ require 'shoryuken/extensions/active_job_adapter'
5
+
6
+ RSpec.describe ActiveJob::QueueAdapters::ShoryukenAdapter::JobWrapper do
7
+ subject { described_class.new }
8
+
9
+ describe '#perform' do
10
+ it 'sets executions to reflect approximate receive count' do
11
+ attributes = { 'ApproximateReceiveCount' => '42' }
12
+ sqs_msg = double Shoryuken::Message, attributes: attributes
13
+ job_hash = { 'arguments' => [1, 2, 3] }
14
+ job_hash_with_executions = { 'arguments' => [1, 2, 3], 'executions' => 41 }
15
+ expect(ActiveJob::Base).to receive(:execute).with(job_hash_with_executions)
16
+
17
+ subject.perform sqs_msg, job_hash
18
+ end
19
+ end
20
+ end
@@ -15,7 +15,7 @@ RSpec.describe Shoryuken::Manager do
15
15
  let(:concurrency) { 1 }
16
16
  let(:executor) { Concurrent::ImmediateExecutor.new }
17
17
 
18
- subject { Shoryuken::Manager.new(fetcher, polling_strategy, concurrency, executor) }
18
+ subject { Shoryuken::Manager.new('default', fetcher, polling_strategy, concurrency, executor) }
19
19
 
20
20
  before do
21
21
  allow(fetcher).to receive(:fetch).and_return([])
@@ -71,6 +71,11 @@ RSpec.describe Shoryuken::Manager do
71
71
 
72
72
  expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
73
73
  expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
74
+ expect(subject).to receive(:fire_event).with(:utilization_update,
75
+ false,
76
+ group: 'default',
77
+ busy_processors: 1,
78
+ max_processors: 1)
74
79
  expect(Shoryuken::Processor).to receive(:process).with(q, message)
75
80
  expect(Shoryuken.logger).to receive(:info).never
76
81
 
@@ -99,6 +104,11 @@ RSpec.describe Shoryuken::Manager do
99
104
  q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
100
105
 
101
106
  expect(fetcher).to receive(:fetch).with(q, described_class::BATCH_LIMIT).and_return(messages)
107
+ expect(subject).to receive(:fire_event).with(:utilization_update,
108
+ false,
109
+ group: 'default',
110
+ busy_processors: 1,
111
+ max_processors: 1)
102
112
  expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
103
113
  allow(subject).to receive(:batched_queue?).with(q).and_return(true)
104
114
  expect(Shoryuken::Processor).to receive(:process).with(q, messages)
@@ -139,4 +149,28 @@ RSpec.describe Shoryuken::Manager do
139
149
  subject.send(:dispatch_single_messages, q)
140
150
  end
141
151
  end
152
+
153
+ describe '#processor_done' do
154
+ let(:sqs_queue) { double Shoryuken::Queue }
155
+
156
+ before do
157
+ allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
158
+ end
159
+
160
+ context 'when queue.fifo? is true' do
161
+ it 'calls message_processed on strategy' do
162
+ expect(sqs_queue).to receive(:fifo?).and_return(true)
163
+ expect(polling_strategy).to receive(:message_processed).with(queue)
164
+ subject.send(:processor_done, queue)
165
+ end
166
+ end
167
+
168
+ context 'when queue.fifo? is false' do
169
+ it 'does not call message_processed on strategy' do
170
+ expect(sqs_queue).to receive(:fifo?).and_return(false)
171
+ expect(polling_strategy).to_not receive(:message_processed)
172
+ subject.send(:processor_done, queue)
173
+ end
174
+ end
175
+ end
142
176
  end