shoryuken 5.0.6 → 5.2.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (37) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/specs.yml +62 -0
  3. data/.reek.yml +5 -0
  4. data/Appraisals +28 -0
  5. data/CHANGELOG.md +48 -0
  6. data/Gemfile +3 -1
  7. data/README.md +21 -1
  8. data/Rakefile +15 -1
  9. data/bin/cli/sqs.rb +50 -5
  10. data/gemfiles/.gitignore +1 -0
  11. data/gemfiles/rails_4_2.gemfile +20 -0
  12. data/gemfiles/rails_5_2.gemfile +21 -0
  13. data/gemfiles/rails_6_0.gemfile +21 -0
  14. data/gemfiles/rails_6_1.gemfile +21 -0
  15. data/lib/shoryuken.rb +1 -0
  16. data/lib/shoryuken/environment_loader.rb +3 -0
  17. data/lib/shoryuken/extensions/active_job_adapter.rb +25 -18
  18. data/lib/shoryuken/extensions/active_job_extensions.rb +38 -0
  19. data/lib/shoryuken/manager.rb +10 -4
  20. data/lib/shoryuken/polling/base.rb +2 -0
  21. data/lib/shoryuken/polling/strict_priority.rb +6 -0
  22. data/lib/shoryuken/polling/weighted_round_robin.rb +11 -0
  23. data/lib/shoryuken/version.rb +1 -1
  24. data/shoryuken.gemspec +0 -1
  25. data/spec/integration/launcher_spec.rb +29 -2
  26. data/spec/shared_examples_for_active_job.rb +226 -9
  27. data/spec/shoryuken/environment_loader_spec.rb +22 -2
  28. data/spec/shoryuken/extensions/active_job_adapter_spec.rb +1 -1
  29. data/spec/shoryuken/extensions/active_job_base_spec.rb +84 -0
  30. data/spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb +4 -0
  31. data/spec/shoryuken/extensions/active_job_wrapper_spec.rb +20 -0
  32. data/spec/shoryuken/manager_spec.rb +24 -0
  33. data/spec/shoryuken/polling/strict_priority_spec.rb +10 -0
  34. data/spec/shoryuken/polling/weighted_round_robin_spec.rb +10 -0
  35. data/spec/spec_helper.rb +5 -9
  36. metadata +16 -19
  37. data/.travis.yml +0 -30
@@ -0,0 +1,38 @@
1
+ module Shoryuken
2
+ module ActiveJobExtensions
3
+ # Adds an accessor for SQS SendMessage parameters on ActiveJob jobs
4
+ # (instances of ActiveJob::Base). Shoryuken ActiveJob queue adapters use
5
+ # these parameters when enqueueing jobs; other adapters can ignore them.
6
+ module SQSSendMessageParametersAccessor
7
+ extend ActiveSupport::Concern
8
+
9
+ included do
10
+ attr_accessor :sqs_send_message_parameters
11
+ end
12
+ end
13
+
14
+ # Initializes SQS SendMessage parameters on instances of ActiveJobe::Base
15
+ # to the empty hash, and populates it whenever `#enqueue` is called, such
16
+ # as when using ActiveJob::Base.set.
17
+ module SQSSendMessageParametersSupport
18
+ def initialize(*arguments)
19
+ super(*arguments)
20
+ self.sqs_send_message_parameters = {}
21
+ end
22
+ ruby2_keywords(:initialize) if respond_to?(:ruby2_keywords, true)
23
+
24
+ def enqueue(options = {})
25
+ sqs_options = options.extract! :message_attributes,
26
+ :message_system_attributes,
27
+ :message_deduplication_id,
28
+ :message_group_id
29
+ sqs_send_message_parameters.merge! sqs_options
30
+
31
+ super
32
+ end
33
+ end
34
+ end
35
+ end
36
+
37
+ ActiveJob::Base.include Shoryuken::ActiveJobExtensions::SQSSendMessageParametersAccessor
38
+ ActiveJob::Base.prepend Shoryuken::ActiveJobExtensions::SQSSendMessageParametersSupport
@@ -57,8 +57,13 @@ module Shoryuken
57
57
  @max_processors - busy
58
58
  end
59
59
 
60
- def processor_done
60
+ def processor_done(queue)
61
61
  @busy_processors.decrement
62
+ client_queue = Shoryuken::Client.queues(queue)
63
+ return unless client_queue.fifo?
64
+ return unless @polling_strategy.respond_to?(:message_processed)
65
+
66
+ @polling_strategy.message_processed(queue)
62
67
  end
63
68
 
64
69
  def assign(queue_name, sqs_msg)
@@ -68,9 +73,10 @@ module Shoryuken
68
73
 
69
74
  @busy_processors.increment
70
75
 
71
- Concurrent::Promise.execute(
72
- executor: @executor
73
- ) { Processor.process(queue_name, sqs_msg) }.then { processor_done }.rescue { processor_done }
76
+ Concurrent::Promise
77
+ .execute(executor: @executor) { Processor.process(queue_name, sqs_msg) }
78
+ .then { processor_done(queue_name) }
79
+ .rescue { processor_done(queue_name) }
74
80
  end
75
81
 
76
82
  def dispatch_batch(queue)
@@ -40,6 +40,8 @@ module Shoryuken
40
40
  fail NotImplementedError
41
41
  end
42
42
 
43
+ def message_processed(_queue); end
44
+
43
45
  def active_queues
44
46
  fail NotImplementedError
45
47
  end
@@ -38,6 +38,11 @@ module Shoryuken
38
38
  .reverse
39
39
  end
40
40
 
41
+ def message_processed(queue)
42
+ logger.debug "Unpausing #{queue}"
43
+ @paused_until[queue] = Time.now
44
+ end
45
+
41
46
  private
42
47
 
43
48
  def next_active_queue
@@ -70,6 +75,7 @@ module Shoryuken
70
75
 
71
76
  def pause(queue)
72
77
  return unless delay > 0
78
+
73
79
  @paused_until[queue] = Time.now + delay
74
80
  logger.debug "Paused #{queue}"
75
81
  end
@@ -35,10 +35,20 @@ module Shoryuken
35
35
  unparse_queues(@queues)
36
36
  end
37
37
 
38
+ def message_processed(queue)
39
+ return if @paused_queues.empty?
40
+
41
+ logger.debug "Unpausing #{queue}"
42
+ @paused_queues.reject! { |_time, name| name == queue }
43
+ @queues << queue
44
+ @queues.uniq!
45
+ end
46
+
38
47
  private
39
48
 
40
49
  def pause(queue)
41
50
  return unless @queues.delete(queue)
51
+
42
52
  @paused_queues << [Time.now + delay, queue]
43
53
  logger.debug "Paused #{queue}"
44
54
  end
@@ -46,6 +56,7 @@ module Shoryuken
46
56
  def unpause_queues
47
57
  return if @paused_queues.empty?
48
58
  return if Time.now < @paused_queues.first[0]
59
+
49
60
  pause = @paused_queues.shift
50
61
  @queues << pause[1]
51
62
  logger.debug "Unpaused #{pause[1]}"
@@ -1,3 +1,3 @@
1
1
  module Shoryuken
2
- VERSION = '5.0.6'.freeze
2
+ VERSION = '5.2.2'.freeze
3
3
  end
data/shoryuken.gemspec CHANGED
@@ -18,7 +18,6 @@ Gem::Specification.new do |spec|
18
18
  spec.require_paths = ['lib']
19
19
 
20
20
  spec.add_development_dependency 'dotenv'
21
- spec.add_development_dependency 'pry-byebug', '3.9.0'
22
21
  spec.add_development_dependency 'rake'
23
22
  spec.add_development_dependency 'rspec'
24
23
 
@@ -4,10 +4,37 @@ require 'shoryuken/launcher'
4
4
  require 'securerandom'
5
5
 
6
6
  RSpec.describe Shoryuken::Launcher do
7
- describe 'Consuming messages', slow: true do
7
+ let(:sqs_client) do
8
+ Aws::SQS::Client.new(
9
+ region: 'us-east-1',
10
+ endpoint: 'http://localhost:5000',
11
+ access_key_id: 'fake',
12
+ secret_access_key: 'fake'
13
+ )
14
+ end
15
+
16
+ let(:executor) do
17
+ # We can't use Concurrent.global_io_executor in these tests since once you
18
+ # shut down a thread pool, you can't start it back up. Instead, we create
19
+ # one new thread pool executor for each spec. We use a new
20
+ # CachedThreadPool, since that most closely resembles
21
+ # Concurrent.global_io_executor
22
+ Concurrent::CachedThreadPool.new auto_terminate: true
23
+ end
24
+
25
+ describe 'Consuming messages' do
8
26
  before do
9
27
  Aws.config[:stub_responses] = false
10
- Aws.config[:region] = 'us-east-1'
28
+
29
+ allow(Shoryuken).to receive(:launcher_executor).and_return(executor)
30
+
31
+ Shoryuken.configure_client do |config|
32
+ config.sqs_client = sqs_client
33
+ end
34
+
35
+ Shoryuken.configure_server do |config|
36
+ config.sqs_client = sqs_client
37
+ end
11
38
 
12
39
  StandardWorker.received_messages = 0
13
40
 
@@ -1,30 +1,44 @@
1
+ require 'active_job'
2
+ require 'shoryuken/extensions/active_job_extensions'
3
+
4
+ # Stand-in for a job class specified by the user
5
+ class TestJob < ActiveJob::Base; end
6
+
1
7
  # rubocop:disable Metrics/BlockLength
2
8
  RSpec.shared_examples 'active_job_adapters' do
3
- let(:job) { double 'Job', id: '123', queue_name: 'queue' }
9
+ let(:job_sqs_send_message_parameters) { {} }
10
+ let(:job) do
11
+ job = TestJob.new
12
+ job.sqs_send_message_parameters = job_sqs_send_message_parameters
13
+ job
14
+ end
4
15
  let(:fifo) { false }
5
16
  let(:queue) { double 'Queue', fifo?: fifo }
6
17
 
7
18
  before do
8
19
  allow(Shoryuken::Client).to receive(:queues).with(job.queue_name).and_return(queue)
9
- allow(job).to receive(:serialize).and_return(
10
- 'job_class' => 'Worker',
11
- 'job_id' => job.id,
12
- 'queue_name' => job.queue_name,
13
- 'arguments' => nil,
14
- 'locale' => nil
15
- )
16
20
  end
17
21
 
18
22
  describe '#enqueue' do
19
23
  specify do
20
24
  expect(queue).to receive(:send_message) do |hash|
21
25
  expect(hash[:message_deduplication_id]).to_not be
26
+ expect(hash[:message_attributes]['shoryuken_class'][:string_value]).to eq(described_class::JobWrapper.to_s)
27
+ expect(hash[:message_attributes]['shoryuken_class'][:data_type]).to eq("String")
28
+ expect(hash[:message_attributes].keys).to eq(['shoryuken_class'])
22
29
  end
23
30
  expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
24
31
 
25
32
  subject.enqueue(job)
26
33
  end
27
34
 
35
+ it "should mutate the job's sqs_send_message_parameters reference to match those sent to the queue" do
36
+ expect(queue).to receive(:send_message) do |options|
37
+ expect(options).to be(job.sqs_send_message_parameters)
38
+ end
39
+ subject.enqueue(job)
40
+ end
41
+
28
42
  context 'when fifo' do
29
43
  let(:fifo) { true }
30
44
 
@@ -38,6 +52,209 @@ RSpec.shared_examples 'active_job_adapters' do
38
52
 
39
53
  subject.enqueue(job)
40
54
  end
55
+
56
+ context 'with message_deduplication_id' do
57
+ context 'when message_deduplication_id is specified in options' do
58
+ it 'should enqueue a message with the deduplication_id specified in options' do
59
+ expect(queue).to receive(:send_message) do |hash|
60
+ expect(hash[:message_deduplication_id]).to eq('options-dedupe-id')
61
+ end
62
+ subject.enqueue(job, message_deduplication_id: 'options-dedupe-id')
63
+ end
64
+ end
65
+
66
+ context 'when message_deduplication_id is specified on the job' do
67
+ let(:job_sqs_send_message_parameters) { { message_deduplication_id: 'job-dedupe-id' } }
68
+
69
+ it 'should enqueue a message with the deduplication_id specified on the job' do
70
+ expect(queue).to receive(:send_message) do |hash|
71
+ expect(hash[:message_deduplication_id]).to eq('job-dedupe-id')
72
+ end
73
+ subject.enqueue job
74
+ end
75
+ end
76
+
77
+ context 'when message_deduplication_id is specified on the job and also in options' do
78
+ let(:job_sqs_send_message_parameters) { { message_deduplication_id: 'job-dedupe-id' } }
79
+
80
+ it 'should enqueue a message with the deduplication_id specified in options' do
81
+ expect(queue).to receive(:send_message) do |hash|
82
+ expect(hash[:message_deduplication_id]).to eq('options-dedupe-id')
83
+ end
84
+ subject.enqueue(job, message_deduplication_id: 'options-dedupe-id')
85
+ end
86
+ end
87
+ end
88
+ end
89
+
90
+ context 'with message_group_id' do
91
+ context 'when message_group_id is specified in options' do
92
+ it 'should enqueue a message with the group_id specified in options' do
93
+ expect(queue).to receive(:send_message) do |hash|
94
+ expect(hash[:message_group_id]).to eq('options-group-id')
95
+ end
96
+ subject.enqueue(job, message_group_id: 'options-group-id')
97
+ end
98
+ end
99
+
100
+ context 'when message_group_id is specified on the job' do
101
+ let(:job_sqs_send_message_parameters) { { message_group_id: 'job-group-id' } }
102
+
103
+ it 'should enqueue a message with the group_id specified on the job' do
104
+ expect(queue).to receive(:send_message) do |hash|
105
+ expect(hash[:message_group_id]).to eq('job-group-id')
106
+ end
107
+ subject.enqueue job
108
+ end
109
+ end
110
+
111
+ context 'when message_group_id is specified on the job and also in options' do
112
+ let(:job_sqs_send_message_parameters) { { message_group_id: 'job-group-id' } }
113
+
114
+ it 'should enqueue a message with the group_id specified in options' do
115
+ expect(queue).to receive(:send_message) do |hash|
116
+ expect(hash[:message_group_id]).to eq('options-group-id')
117
+ end
118
+ subject.enqueue(job, message_group_id: 'options-group-id')
119
+ end
120
+ end
121
+ end
122
+
123
+ context 'with additional message attributes' do
124
+ it 'should combine with activejob attributes' do
125
+ custom_message_attributes = {
126
+ 'tracer_id' => {
127
+ string_value: SecureRandom.hex,
128
+ data_type: 'String'
129
+ }
130
+ }
131
+
132
+ expect(queue).to receive(:send_message) do |hash|
133
+ expect(hash[:message_attributes]['shoryuken_class'][:string_value]).to eq(described_class::JobWrapper.to_s)
134
+ expect(hash[:message_attributes]['shoryuken_class'][:data_type]).to eq("String")
135
+ expect(hash[:message_attributes]['tracer_id'][:string_value]).to eq(custom_message_attributes['tracer_id'][:string_value])
136
+ expect(hash[:message_attributes]['tracer_id'][:data_type]).to eq("String")
137
+ end
138
+ expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
139
+
140
+ subject.enqueue(job, message_attributes: custom_message_attributes)
141
+ end
142
+
143
+ context 'when message_attributes are specified on the job' do
144
+ let(:job_sqs_send_message_parameters) do
145
+ {
146
+ message_attributes: {
147
+ 'tracer_id' => {
148
+ data_type: 'String',
149
+ string_value: 'job-value'
150
+ }
151
+ }
152
+ }
153
+ end
154
+
155
+ it 'should enqueue a message with the message_attributes specified on the job' do
156
+ expect(queue).to receive(:send_message) do |hash|
157
+ expect(hash[:message_attributes]['tracer_id']).to eq({ data_type: 'String', string_value: 'job-value' })
158
+ expect(hash[:message_attributes]['shoryuken_class']).to eq({ data_type: 'String', string_value: described_class::JobWrapper.to_s })
159
+ end
160
+ subject.enqueue job
161
+ end
162
+ end
163
+
164
+ context 'when message_attributes are specified on the job and also in options' do
165
+ let(:job_sqs_send_message_parameters) do
166
+ {
167
+ message_attributes: {
168
+ 'tracer_id' => {
169
+ data_type: 'String',
170
+ string_value: 'job-value'
171
+ }
172
+ }
173
+ }
174
+ end
175
+
176
+ it 'should enqueue a message with the message_attributes speficied in options' do
177
+ custom_message_attributes = {
178
+ 'options_tracer_id' => {
179
+ string_value: 'options-value',
180
+ data_type: 'String'
181
+ }
182
+ }
183
+
184
+ expect(queue).to receive(:send_message) do |hash|
185
+ expect(hash[:message_attributes]['tracer_id']).to be_nil
186
+ expect(hash[:message_attributes]['options_tracer_id']).to eq({ data_type: 'String', string_value: 'options-value' })
187
+ expect(hash[:message_attributes]['shoryuken_class']).to eq({ data_type: 'String', string_value: described_class::JobWrapper.to_s })
188
+ end
189
+ subject.enqueue job, message_attributes: custom_message_attributes
190
+ end
191
+ end
192
+ end
193
+ end
194
+
195
+ context 'with message_system_attributes' do
196
+ context 'when message_system_attributes are specified in options' do
197
+ it 'should enqueue a message with message_system_attributes specified in options' do
198
+ system_attributes = {
199
+ 'AWSTraceHeader' => {
200
+ string_value: 'trace_id',
201
+ data_type: 'String'
202
+ }
203
+ }
204
+ expect(queue).to receive(:send_message) do |hash|
205
+ expect(hash[:message_system_attributes]['AWSTraceHeader'][:string_value]).to eq('trace_id')
206
+ expect(hash[:message_system_attributes]['AWSTraceHeader'][:data_type]).to eq('String')
207
+ end
208
+ subject.enqueue(job, message_system_attributes: system_attributes)
209
+ end
210
+ end
211
+
212
+ context 'when message_system_attributes are specified on the job' do
213
+ let(:job_sqs_send_message_parameters) do
214
+ {
215
+ message_system_attributes: {
216
+ 'AWSTraceHeader' => {
217
+ string_value: 'job-value',
218
+ data_type: 'String'
219
+ }
220
+ }
221
+ }
222
+ end
223
+
224
+ it 'should enqueue a message with the message_system_attributes specified on the job' do
225
+ expect(queue).to receive(:send_message) do |hash|
226
+ expect(hash[:message_system_attributes]['AWSTraceHeader']).to eq({ data_type: 'String', string_value: 'job-value' })
227
+ end
228
+ subject.enqueue job
229
+ end
230
+ end
231
+
232
+ context 'when message_system_attributes are specified on the job and also in options' do
233
+ let(:job_sqs_send_message_parameters) do
234
+ {
235
+ message_system_attributes: {
236
+ 'job_trace_header' => {
237
+ string_value: 'job-value',
238
+ data_type: 'String'
239
+ }
240
+ }
241
+ }
242
+ end
243
+
244
+ it 'should enqueue a message with the message_system_attributes speficied in options' do
245
+ custom_message_attributes = {
246
+ 'options_trace_header' => {
247
+ string_value: 'options-value',
248
+ data_type: 'String'
249
+ }
250
+ }
251
+
252
+ expect(queue).to receive(:send_message) do |hash|
253
+ expect(hash[:message_system_attributes]['job_trace_header']).to be_nil
254
+ expect(hash[:message_system_attributes]['options_trace_header']).to eq({ data_type: 'String', string_value: 'options-value' })
255
+ end
256
+ subject.enqueue job, message_system_attributes: custom_message_attributes
257
+ end
41
258
  end
42
259
  end
43
260
 
@@ -59,4 +276,4 @@ RSpec.shared_examples 'active_job_adapters' do
59
276
  end
60
277
  end
61
278
  end
62
- # rubocop:enable Metrics/BlockLength
279
+ # rubocop:enable Metrics/BlockLength