shoryuken 5.0.2 → 5.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 177f42239f7942853a00ec1c527a63ef53662442e03c628f6b01f0a8a60f3c61
4
- data.tar.gz: 6a6872a321ad6e300101905fbc08bc76dbcdc4a160d05eb8e061ae1972b2de67
3
+ metadata.gz: 64491bd4386735606adc6883e2f9118c8d14eb996f3e6f0eff54cd567a362333
4
+ data.tar.gz: 2257c6730d6f6dd687a2e72246670977e8bea00d5e4d1259d935a57647ffadad
5
5
  SHA512:
6
- metadata.gz: fcb645e36ec34051bcc588799dec40b3a7a0f69982c53ba8f9f309ebcf1db7c8b69d37f5c3765a9f7aa27ea710631fefa868241f253dc3a26400a77aa73a82fa
7
- data.tar.gz: df7c1991c8e5370b4fe789ce2e0fa23c2270a12de6c40cf2ac5385e4587e5bc2259d90cb12c00a974bfc9bd7983ca7557927ad6c88e7bbd4fcf00481bc3b691d
6
+ metadata.gz: dea5b0b639fcfc604b5179bed637755a70e2eb6670ab62895c0fcbda45931bab7dedf15d3d2b03877351080d8ed98c236e2c10d718addb573d9dcf0ec735d679
7
+ data.tar.gz: 5b7f057295393124a48e7eb1c4ebf2f360f5e05fe496f7710d3ee7320331e2c2184e2a06dd9c737ecec1022d9a16f826f0f2ef6fd01276776a0359cdf49036f5
@@ -32,7 +32,7 @@ Style/Alias:
32
32
  Style/PerlBackrefs:
33
33
  Enabled: false
34
34
 
35
- Layout/TrailingBlankLines:
35
+ Layout/TrailingEmptyLines:
36
36
  Enabled: false
37
37
 
38
38
  # Override the HoundCI custom rules (they do not use Rubocop defaults)
@@ -76,7 +76,7 @@ Style/GuardClause:
76
76
  Style/RegexpLiteral:
77
77
  Enabled: false
78
78
 
79
- Lint/HandleExceptions:
79
+ Lint/SuppressedException:
80
80
  Enabled: false
81
81
 
82
82
  Lint/AssignmentInCondition:
@@ -117,6 +117,3 @@ Security/YAMLLoad:
117
117
 
118
118
  Naming/MemoizedInstanceVariableName:
119
119
  Enabled: false
120
-
121
- Performance/RedundantBlockCall:
122
- Enabled: false
@@ -1,3 +1,9 @@
1
+ ## [v5.0.3] - 2019-11-30
2
+
3
+ - Add support for sending messages asynchronous with Active Job using `shoryuken_concurrent_send`
4
+ - [#589](https://github.com/phstc/shoryuken/pull/589)
5
+ - [#588](https://github.com/phstc/shoryuken/pull/588)
6
+
1
7
  ## [v5.0.2] - 2019-11-02
2
8
 
3
9
  - Fix Queue order is reversed if passed through CLI
data/README.md CHANGED
@@ -37,7 +37,7 @@ If you are using AWS SDK version 3, please also add this line:
37
37
  gem 'aws-sdk-sqs'
38
38
  ```
39
39
 
40
- The extra gem `aws-sdk-sqs` is required in order to keep Shoryuken compatible with AWS SDK version 2 and 3.
40
+ The extra gem `aws-sdk-sqs` is required in order to keep Shoryuken compatible with AWS SDK version 2 and 3.
41
41
 
42
42
  And then execute:
43
43
 
@@ -88,4 +88,7 @@ module Shoryuken
88
88
  )
89
89
  end
90
90
 
91
- require 'shoryuken/extensions/active_job_adapter' if Shoryuken.active_job?
91
+ if Shoryuken.active_job?
92
+ require 'shoryuken/extensions/active_job_adapter'
93
+ require 'shoryuken/extensions/active_job_concurrent_send_adapter'
94
+ end
@@ -1,7 +1,7 @@
1
1
  module Shoryuken
2
2
  class DefaultWorkerRegistry < WorkerRegistry
3
3
  def initialize
4
- @workers = {}
4
+ @workers = Concurrent::Hash.new
5
5
  end
6
6
 
7
7
  def batch_receive_messages?(queue)
@@ -0,0 +1,50 @@
1
+ # ActiveJob docs: http://edgeguides.rubyonrails.org/active_job_basics.html
2
+ # Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
3
+ module ActiveJob
4
+ module QueueAdapters
5
+ # == Shoryuken concurrent adapter for Active Job
6
+ #
7
+ # This adapter sends messages asynchronously (ie non-blocking) and allows
8
+ # the caller to set up handlers for both success and failure
9
+ #
10
+ # To use this adapter, set up as:
11
+ #
12
+ # success_handler = ->(response, job, options) { StatsD.increment("#{job.class.name}.success") }
13
+ # error_handler = ->(err, job, options) { StatsD.increment("#{job.class.name}.failure") }
14
+ #
15
+ # adapter = ActiveJob::QueueAdapters::ShoryukenConcurrentSendAdapter.new(success_handler, error_handler)
16
+ #
17
+ # config.active_job.queue_adapter = adapter
18
+ class ShoryukenConcurrentSendAdapter < ShoryukenAdapter
19
+ def initialize(success_handler = nil, error_handler = nil)
20
+ @success_handler = success_handler
21
+ @error_handler = error_handler
22
+ end
23
+
24
+ def enqueue(job, options = {})
25
+ send_concurrently(job, options) { |f_job, f_options| super(f_job, f_options) }
26
+ end
27
+
28
+ def success_handler
29
+ @success_handler ||= ->(_send_message_response, _job, _options) { nil }
30
+ end
31
+
32
+ def error_handler
33
+ @error_handler ||= begin
34
+ lambda { |error, job, _options|
35
+ Shoryuken.logger.warn("Failed to enqueue job: #{job.inspect} due to error: #{error}")
36
+ }
37
+ end
38
+ end
39
+
40
+ private
41
+
42
+ def send_concurrently(job, options)
43
+ Concurrent::Promises
44
+ .future(job, options) { |f_job, f_options| [yield(f_job, f_options), f_job, f_options] }
45
+ .then { |send_message_response, f_job, f_options| success_handler.call(send_message_response, f_job, f_options) }
46
+ .rescue(job, options) { |err, f_job, f_options| error_handler.call(err, f_job, f_options) }
47
+ end
48
+ end
49
+ end
50
+ end
@@ -1,3 +1,3 @@
1
1
  module Shoryuken
2
- VERSION = '5.0.2'.freeze
2
+ VERSION = '5.0.3'.freeze
3
3
  end
@@ -0,0 +1,62 @@
1
+ # rubocop:disable Metrics/BlockLength
2
+ RSpec.shared_examples 'active_job_adapters' do
3
+ let(:job) { double 'Job', id: '123', queue_name: 'queue' }
4
+ let(:fifo) { false }
5
+ let(:queue) { double 'Queue', fifo?: fifo }
6
+
7
+ before do
8
+ allow(Shoryuken::Client).to receive(:queues).with(job.queue_name).and_return(queue)
9
+ allow(job).to receive(:serialize).and_return(
10
+ 'job_class' => 'Worker',
11
+ 'job_id' => job.id,
12
+ 'queue_name' => job.queue_name,
13
+ 'arguments' => nil,
14
+ 'locale' => nil
15
+ )
16
+ end
17
+
18
+ describe '#enqueue' do
19
+ specify do
20
+ expect(queue).to receive(:send_message) do |hash|
21
+ expect(hash[:message_deduplication_id]).to_not be
22
+ end
23
+ expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
24
+
25
+ subject.enqueue(job)
26
+ end
27
+
28
+ context 'when fifo' do
29
+ let(:fifo) { true }
30
+
31
+ it 'does not include job_id in the deduplication_id' do
32
+ expect(queue).to receive(:send_message) do |hash|
33
+ message_deduplication_id = Digest::SHA256.hexdigest(JSON.dump(job.serialize.except('job_id')))
34
+
35
+ expect(hash[:message_deduplication_id]).to eq(message_deduplication_id)
36
+ end
37
+ expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
38
+
39
+ subject.enqueue(job)
40
+ end
41
+ end
42
+ end
43
+
44
+ describe '#enqueue_at' do
45
+ specify do
46
+ delay = 1
47
+
48
+ expect(queue).to receive(:send_message) do |hash|
49
+ expect(hash[:message_deduplication_id]).to_not be
50
+ expect(hash[:delay_seconds]).to eq(delay)
51
+ end
52
+
53
+ expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
54
+
55
+ # need to figure out what to require Time.current and N.minutes to remove the stub
56
+ allow(subject).to receive(:calculate_delay).and_return(delay)
57
+
58
+ subject.enqueue_at(job, nil)
59
+ end
60
+ end
61
+ end
62
+ # rubocop:enable Metrics/BlockLength
@@ -1,64 +1,7 @@
1
1
  require 'spec_helper'
2
- require 'active_job'
3
2
  require 'shoryuken/extensions/active_job_adapter'
3
+ require 'shared_examples_for_active_job'
4
4
 
5
5
  RSpec.describe ActiveJob::QueueAdapters::ShoryukenAdapter do
6
- let(:job) { double 'Job', id: '123', queue_name: 'queue' }
7
- let(:fifo) { false }
8
- let(:queue) { double 'Queue', fifo?: fifo }
9
-
10
- before do
11
- allow(Shoryuken::Client).to receive(:queues).with(job.queue_name).and_return(queue)
12
- allow(job).to receive(:serialize).and_return(
13
- 'job_class' => 'Worker',
14
- 'job_id' => job.id,
15
- 'queue_name' => job.queue_name,
16
- 'arguments' => nil,
17
- 'locale' => nil
18
- )
19
- end
20
-
21
- describe '#enqueue' do
22
- specify do
23
- expect(queue).to receive(:send_message) do |hash|
24
- expect(hash[:message_deduplication_id]).to_not be
25
- end
26
- expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
27
-
28
- subject.enqueue(job)
29
- end
30
-
31
- context 'when fifo' do
32
- let(:fifo) { true }
33
-
34
- it 'does not include job_id in the deduplication_id' do
35
- expect(queue).to receive(:send_message) do |hash|
36
- message_deduplication_id = Digest::SHA256.hexdigest(JSON.dump(job.serialize.except('job_id')))
37
-
38
- expect(hash[:message_deduplication_id]).to eq(message_deduplication_id)
39
- end
40
- expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
41
-
42
- subject.enqueue(job)
43
- end
44
- end
45
- end
46
-
47
- describe '#enqueue_at' do
48
- specify do
49
- delay = 1
50
-
51
- expect(queue).to receive(:send_message) do |hash|
52
- expect(hash[:message_deduplication_id]).to_not be
53
- expect(hash[:delay_seconds]).to eq(delay)
54
- end
55
-
56
- expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
57
-
58
- # need to figure out what to require Time.current and N.minutes to remove the stub
59
- allow(subject).to receive(:calculate_delay).and_return(delay)
60
-
61
- subject.enqueue_at(job, nil)
62
- end
63
- end
6
+ include_examples 'active_job_adapters'
64
7
  end
@@ -0,0 +1,35 @@
1
+ require 'spec_helper'
2
+ require 'shared_examples_for_active_job'
3
+ require 'shoryuken/extensions/active_job_adapter'
4
+ require 'shoryuken/extensions/active_job_concurrent_send_adapter'
5
+
6
+ RSpec.describe ActiveJob::QueueAdapters::ShoryukenConcurrentSendAdapter do
7
+ include_examples 'active_job_adapters'
8
+
9
+ let(:options) { {} }
10
+ let(:error_handler) { -> {} }
11
+ let(:success_handler) { -> {} }
12
+
13
+ subject { described_class.new(success_handler, error_handler) }
14
+
15
+ context 'when success' do
16
+ it 'calls success_handler' do
17
+ response = true
18
+ allow(queue).to receive(:send_message).and_return(response)
19
+ expect(success_handler).to receive(:call).with(response, job, options)
20
+
21
+ subject.enqueue(job, options)
22
+ end
23
+ end
24
+
25
+ context 'when failure' do
26
+ it 'calls error_handler' do
27
+ response = Aws::SQS::Errors::InternalError.new('error', 'error')
28
+
29
+ allow(queue).to receive(:send_message).and_raise(response)
30
+ expect(error_handler).to receive(:call).with(response, job, options).and_call_original
31
+
32
+ subject.enqueue(job, options)
33
+ end
34
+ end
35
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: shoryuken
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.0.2
4
+ version: 5.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Pablo Cantero
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-11-02 00:00:00.000000000 Z
11
+ date: 2019-12-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: dotenv
@@ -140,6 +140,7 @@ files:
140
140
  - lib/shoryuken/default_worker_registry.rb
141
141
  - lib/shoryuken/environment_loader.rb
142
142
  - lib/shoryuken/extensions/active_job_adapter.rb
143
+ - lib/shoryuken/extensions/active_job_concurrent_send_adapter.rb
143
144
  - lib/shoryuken/fetcher.rb
144
145
  - lib/shoryuken/launcher.rb
145
146
  - lib/shoryuken/logging.rb
@@ -167,6 +168,7 @@ files:
167
168
  - shoryuken.gemspec
168
169
  - shoryuken.jpg
169
170
  - spec/integration/launcher_spec.rb
171
+ - spec/shared_examples_for_active_job.rb
170
172
  - spec/shoryuken.yml
171
173
  - spec/shoryuken/body_parser_spec.rb
172
174
  - spec/shoryuken/client_spec.rb
@@ -174,6 +176,7 @@ files:
174
176
  - spec/shoryuken/default_worker_registry_spec.rb
175
177
  - spec/shoryuken/environment_loader_spec.rb
176
178
  - spec/shoryuken/extensions/active_job_adapter_spec.rb
179
+ - spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb
177
180
  - spec/shoryuken/fetcher_spec.rb
178
181
  - spec/shoryuken/manager_spec.rb
179
182
  - spec/shoryuken/middleware/chain_spec.rb
@@ -215,12 +218,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
215
218
  version: '0'
216
219
  requirements: []
217
220
  rubyforge_project:
218
- rubygems_version: 2.7.6.2
221
+ rubygems_version: 2.7.6
219
222
  signing_key:
220
223
  specification_version: 4
221
224
  summary: Shoryuken is a super efficient AWS SQS thread based message processor
222
225
  test_files:
223
226
  - spec/integration/launcher_spec.rb
227
+ - spec/shared_examples_for_active_job.rb
224
228
  - spec/shoryuken.yml
225
229
  - spec/shoryuken/body_parser_spec.rb
226
230
  - spec/shoryuken/client_spec.rb
@@ -228,6 +232,7 @@ test_files:
228
232
  - spec/shoryuken/default_worker_registry_spec.rb
229
233
  - spec/shoryuken/environment_loader_spec.rb
230
234
  - spec/shoryuken/extensions/active_job_adapter_spec.rb
235
+ - spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb
231
236
  - spec/shoryuken/fetcher_spec.rb
232
237
  - spec/shoryuken/manager_spec.rb
233
238
  - spec/shoryuken/middleware/chain_spec.rb