shoryuken 5.0.2 → 5.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -5
- data/CHANGELOG.md +6 -0
- data/README.md +1 -1
- data/lib/shoryuken.rb +4 -1
- data/lib/shoryuken/default_worker_registry.rb +1 -1
- data/lib/shoryuken/extensions/active_job_concurrent_send_adapter.rb +50 -0
- data/lib/shoryuken/version.rb +1 -1
- data/spec/shared_examples_for_active_job.rb +62 -0
- data/spec/shoryuken/extensions/active_job_adapter_spec.rb +2 -59
- data/spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb +35 -0
- metadata +8 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 64491bd4386735606adc6883e2f9118c8d14eb996f3e6f0eff54cd567a362333
|
4
|
+
data.tar.gz: 2257c6730d6f6dd687a2e72246670977e8bea00d5e4d1259d935a57647ffadad
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dea5b0b639fcfc604b5179bed637755a70e2eb6670ab62895c0fcbda45931bab7dedf15d3d2b03877351080d8ed98c236e2c10d718addb573d9dcf0ec735d679
|
7
|
+
data.tar.gz: 5b7f057295393124a48e7eb1c4ebf2f360f5e05fe496f7710d3ee7320331e2c2184e2a06dd9c737ecec1022d9a16f826f0f2ef6fd01276776a0359cdf49036f5
|
data/.rubocop.yml
CHANGED
@@ -32,7 +32,7 @@ Style/Alias:
|
|
32
32
|
Style/PerlBackrefs:
|
33
33
|
Enabled: false
|
34
34
|
|
35
|
-
Layout/
|
35
|
+
Layout/TrailingEmptyLines:
|
36
36
|
Enabled: false
|
37
37
|
|
38
38
|
# Override the HoundCI custom rules (they do not use Rubocop defaults)
|
@@ -76,7 +76,7 @@ Style/GuardClause:
|
|
76
76
|
Style/RegexpLiteral:
|
77
77
|
Enabled: false
|
78
78
|
|
79
|
-
Lint/
|
79
|
+
Lint/SuppressedException:
|
80
80
|
Enabled: false
|
81
81
|
|
82
82
|
Lint/AssignmentInCondition:
|
@@ -117,6 +117,3 @@ Security/YAMLLoad:
|
|
117
117
|
|
118
118
|
Naming/MemoizedInstanceVariableName:
|
119
119
|
Enabled: false
|
120
|
-
|
121
|
-
Performance/RedundantBlockCall:
|
122
|
-
Enabled: false
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
## [v5.0.3] - 2019-11-30
|
2
|
+
|
3
|
+
- Add support for sending messages asynchronous with Active Job using `shoryuken_concurrent_send`
|
4
|
+
- [#589](https://github.com/phstc/shoryuken/pull/589)
|
5
|
+
- [#588](https://github.com/phstc/shoryuken/pull/588)
|
6
|
+
|
1
7
|
## [v5.0.2] - 2019-11-02
|
2
8
|
|
3
9
|
- Fix Queue order is reversed if passed through CLI
|
data/README.md
CHANGED
@@ -37,7 +37,7 @@ If you are using AWS SDK version 3, please also add this line:
|
|
37
37
|
gem 'aws-sdk-sqs'
|
38
38
|
```
|
39
39
|
|
40
|
-
The extra gem `aws-sdk-sqs` is required in order to keep Shoryuken compatible with AWS SDK version 2 and 3.
|
40
|
+
The extra gem `aws-sdk-sqs` is required in order to keep Shoryuken compatible with AWS SDK version 2 and 3.
|
41
41
|
|
42
42
|
And then execute:
|
43
43
|
|
data/lib/shoryuken.rb
CHANGED
@@ -88,4 +88,7 @@ module Shoryuken
|
|
88
88
|
)
|
89
89
|
end
|
90
90
|
|
91
|
-
|
91
|
+
if Shoryuken.active_job?
|
92
|
+
require 'shoryuken/extensions/active_job_adapter'
|
93
|
+
require 'shoryuken/extensions/active_job_concurrent_send_adapter'
|
94
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# ActiveJob docs: http://edgeguides.rubyonrails.org/active_job_basics.html
|
2
|
+
# Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
|
3
|
+
module ActiveJob
|
4
|
+
module QueueAdapters
|
5
|
+
# == Shoryuken concurrent adapter for Active Job
|
6
|
+
#
|
7
|
+
# This adapter sends messages asynchronously (ie non-blocking) and allows
|
8
|
+
# the caller to set up handlers for both success and failure
|
9
|
+
#
|
10
|
+
# To use this adapter, set up as:
|
11
|
+
#
|
12
|
+
# success_handler = ->(response, job, options) { StatsD.increment("#{job.class.name}.success") }
|
13
|
+
# error_handler = ->(err, job, options) { StatsD.increment("#{job.class.name}.failure") }
|
14
|
+
#
|
15
|
+
# adapter = ActiveJob::QueueAdapters::ShoryukenConcurrentSendAdapter.new(success_handler, error_handler)
|
16
|
+
#
|
17
|
+
# config.active_job.queue_adapter = adapter
|
18
|
+
class ShoryukenConcurrentSendAdapter < ShoryukenAdapter
|
19
|
+
def initialize(success_handler = nil, error_handler = nil)
|
20
|
+
@success_handler = success_handler
|
21
|
+
@error_handler = error_handler
|
22
|
+
end
|
23
|
+
|
24
|
+
def enqueue(job, options = {})
|
25
|
+
send_concurrently(job, options) { |f_job, f_options| super(f_job, f_options) }
|
26
|
+
end
|
27
|
+
|
28
|
+
def success_handler
|
29
|
+
@success_handler ||= ->(_send_message_response, _job, _options) { nil }
|
30
|
+
end
|
31
|
+
|
32
|
+
def error_handler
|
33
|
+
@error_handler ||= begin
|
34
|
+
lambda { |error, job, _options|
|
35
|
+
Shoryuken.logger.warn("Failed to enqueue job: #{job.inspect} due to error: #{error}")
|
36
|
+
}
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def send_concurrently(job, options)
|
43
|
+
Concurrent::Promises
|
44
|
+
.future(job, options) { |f_job, f_options| [yield(f_job, f_options), f_job, f_options] }
|
45
|
+
.then { |send_message_response, f_job, f_options| success_handler.call(send_message_response, f_job, f_options) }
|
46
|
+
.rescue(job, options) { |err, f_job, f_options| error_handler.call(err, f_job, f_options) }
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
data/lib/shoryuken/version.rb
CHANGED
@@ -0,0 +1,62 @@
|
|
1
|
+
# rubocop:disable Metrics/BlockLength
|
2
|
+
RSpec.shared_examples 'active_job_adapters' do
|
3
|
+
let(:job) { double 'Job', id: '123', queue_name: 'queue' }
|
4
|
+
let(:fifo) { false }
|
5
|
+
let(:queue) { double 'Queue', fifo?: fifo }
|
6
|
+
|
7
|
+
before do
|
8
|
+
allow(Shoryuken::Client).to receive(:queues).with(job.queue_name).and_return(queue)
|
9
|
+
allow(job).to receive(:serialize).and_return(
|
10
|
+
'job_class' => 'Worker',
|
11
|
+
'job_id' => job.id,
|
12
|
+
'queue_name' => job.queue_name,
|
13
|
+
'arguments' => nil,
|
14
|
+
'locale' => nil
|
15
|
+
)
|
16
|
+
end
|
17
|
+
|
18
|
+
describe '#enqueue' do
|
19
|
+
specify do
|
20
|
+
expect(queue).to receive(:send_message) do |hash|
|
21
|
+
expect(hash[:message_deduplication_id]).to_not be
|
22
|
+
end
|
23
|
+
expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
|
24
|
+
|
25
|
+
subject.enqueue(job)
|
26
|
+
end
|
27
|
+
|
28
|
+
context 'when fifo' do
|
29
|
+
let(:fifo) { true }
|
30
|
+
|
31
|
+
it 'does not include job_id in the deduplication_id' do
|
32
|
+
expect(queue).to receive(:send_message) do |hash|
|
33
|
+
message_deduplication_id = Digest::SHA256.hexdigest(JSON.dump(job.serialize.except('job_id')))
|
34
|
+
|
35
|
+
expect(hash[:message_deduplication_id]).to eq(message_deduplication_id)
|
36
|
+
end
|
37
|
+
expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
|
38
|
+
|
39
|
+
subject.enqueue(job)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
describe '#enqueue_at' do
|
45
|
+
specify do
|
46
|
+
delay = 1
|
47
|
+
|
48
|
+
expect(queue).to receive(:send_message) do |hash|
|
49
|
+
expect(hash[:message_deduplication_id]).to_not be
|
50
|
+
expect(hash[:delay_seconds]).to eq(delay)
|
51
|
+
end
|
52
|
+
|
53
|
+
expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
|
54
|
+
|
55
|
+
# need to figure out what to require Time.current and N.minutes to remove the stub
|
56
|
+
allow(subject).to receive(:calculate_delay).and_return(delay)
|
57
|
+
|
58
|
+
subject.enqueue_at(job, nil)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
# rubocop:enable Metrics/BlockLength
|
@@ -1,64 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
-
require 'active_job'
|
3
2
|
require 'shoryuken/extensions/active_job_adapter'
|
3
|
+
require 'shared_examples_for_active_job'
|
4
4
|
|
5
5
|
RSpec.describe ActiveJob::QueueAdapters::ShoryukenAdapter do
|
6
|
-
|
7
|
-
let(:fifo) { false }
|
8
|
-
let(:queue) { double 'Queue', fifo?: fifo }
|
9
|
-
|
10
|
-
before do
|
11
|
-
allow(Shoryuken::Client).to receive(:queues).with(job.queue_name).and_return(queue)
|
12
|
-
allow(job).to receive(:serialize).and_return(
|
13
|
-
'job_class' => 'Worker',
|
14
|
-
'job_id' => job.id,
|
15
|
-
'queue_name' => job.queue_name,
|
16
|
-
'arguments' => nil,
|
17
|
-
'locale' => nil
|
18
|
-
)
|
19
|
-
end
|
20
|
-
|
21
|
-
describe '#enqueue' do
|
22
|
-
specify do
|
23
|
-
expect(queue).to receive(:send_message) do |hash|
|
24
|
-
expect(hash[:message_deduplication_id]).to_not be
|
25
|
-
end
|
26
|
-
expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
|
27
|
-
|
28
|
-
subject.enqueue(job)
|
29
|
-
end
|
30
|
-
|
31
|
-
context 'when fifo' do
|
32
|
-
let(:fifo) { true }
|
33
|
-
|
34
|
-
it 'does not include job_id in the deduplication_id' do
|
35
|
-
expect(queue).to receive(:send_message) do |hash|
|
36
|
-
message_deduplication_id = Digest::SHA256.hexdigest(JSON.dump(job.serialize.except('job_id')))
|
37
|
-
|
38
|
-
expect(hash[:message_deduplication_id]).to eq(message_deduplication_id)
|
39
|
-
end
|
40
|
-
expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
|
41
|
-
|
42
|
-
subject.enqueue(job)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
describe '#enqueue_at' do
|
48
|
-
specify do
|
49
|
-
delay = 1
|
50
|
-
|
51
|
-
expect(queue).to receive(:send_message) do |hash|
|
52
|
-
expect(hash[:message_deduplication_id]).to_not be
|
53
|
-
expect(hash[:delay_seconds]).to eq(delay)
|
54
|
-
end
|
55
|
-
|
56
|
-
expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
|
57
|
-
|
58
|
-
# need to figure out what to require Time.current and N.minutes to remove the stub
|
59
|
-
allow(subject).to receive(:calculate_delay).and_return(delay)
|
60
|
-
|
61
|
-
subject.enqueue_at(job, nil)
|
62
|
-
end
|
63
|
-
end
|
6
|
+
include_examples 'active_job_adapters'
|
64
7
|
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'shared_examples_for_active_job'
|
3
|
+
require 'shoryuken/extensions/active_job_adapter'
|
4
|
+
require 'shoryuken/extensions/active_job_concurrent_send_adapter'
|
5
|
+
|
6
|
+
RSpec.describe ActiveJob::QueueAdapters::ShoryukenConcurrentSendAdapter do
|
7
|
+
include_examples 'active_job_adapters'
|
8
|
+
|
9
|
+
let(:options) { {} }
|
10
|
+
let(:error_handler) { -> {} }
|
11
|
+
let(:success_handler) { -> {} }
|
12
|
+
|
13
|
+
subject { described_class.new(success_handler, error_handler) }
|
14
|
+
|
15
|
+
context 'when success' do
|
16
|
+
it 'calls success_handler' do
|
17
|
+
response = true
|
18
|
+
allow(queue).to receive(:send_message).and_return(response)
|
19
|
+
expect(success_handler).to receive(:call).with(response, job, options)
|
20
|
+
|
21
|
+
subject.enqueue(job, options)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
context 'when failure' do
|
26
|
+
it 'calls error_handler' do
|
27
|
+
response = Aws::SQS::Errors::InternalError.new('error', 'error')
|
28
|
+
|
29
|
+
allow(queue).to receive(:send_message).and_raise(response)
|
30
|
+
expect(error_handler).to receive(:call).with(response, job, options).and_call_original
|
31
|
+
|
32
|
+
subject.enqueue(job, options)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shoryuken
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.0.
|
4
|
+
version: 5.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pablo Cantero
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-12-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: dotenv
|
@@ -140,6 +140,7 @@ files:
|
|
140
140
|
- lib/shoryuken/default_worker_registry.rb
|
141
141
|
- lib/shoryuken/environment_loader.rb
|
142
142
|
- lib/shoryuken/extensions/active_job_adapter.rb
|
143
|
+
- lib/shoryuken/extensions/active_job_concurrent_send_adapter.rb
|
143
144
|
- lib/shoryuken/fetcher.rb
|
144
145
|
- lib/shoryuken/launcher.rb
|
145
146
|
- lib/shoryuken/logging.rb
|
@@ -167,6 +168,7 @@ files:
|
|
167
168
|
- shoryuken.gemspec
|
168
169
|
- shoryuken.jpg
|
169
170
|
- spec/integration/launcher_spec.rb
|
171
|
+
- spec/shared_examples_for_active_job.rb
|
170
172
|
- spec/shoryuken.yml
|
171
173
|
- spec/shoryuken/body_parser_spec.rb
|
172
174
|
- spec/shoryuken/client_spec.rb
|
@@ -174,6 +176,7 @@ files:
|
|
174
176
|
- spec/shoryuken/default_worker_registry_spec.rb
|
175
177
|
- spec/shoryuken/environment_loader_spec.rb
|
176
178
|
- spec/shoryuken/extensions/active_job_adapter_spec.rb
|
179
|
+
- spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb
|
177
180
|
- spec/shoryuken/fetcher_spec.rb
|
178
181
|
- spec/shoryuken/manager_spec.rb
|
179
182
|
- spec/shoryuken/middleware/chain_spec.rb
|
@@ -215,12 +218,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
215
218
|
version: '0'
|
216
219
|
requirements: []
|
217
220
|
rubyforge_project:
|
218
|
-
rubygems_version: 2.7.6
|
221
|
+
rubygems_version: 2.7.6
|
219
222
|
signing_key:
|
220
223
|
specification_version: 4
|
221
224
|
summary: Shoryuken is a super efficient AWS SQS thread based message processor
|
222
225
|
test_files:
|
223
226
|
- spec/integration/launcher_spec.rb
|
227
|
+
- spec/shared_examples_for_active_job.rb
|
224
228
|
- spec/shoryuken.yml
|
225
229
|
- spec/shoryuken/body_parser_spec.rb
|
226
230
|
- spec/shoryuken/client_spec.rb
|
@@ -228,6 +232,7 @@ test_files:
|
|
228
232
|
- spec/shoryuken/default_worker_registry_spec.rb
|
229
233
|
- spec/shoryuken/environment_loader_spec.rb
|
230
234
|
- spec/shoryuken/extensions/active_job_adapter_spec.rb
|
235
|
+
- spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb
|
231
236
|
- spec/shoryuken/fetcher_spec.rb
|
232
237
|
- spec/shoryuken/manager_spec.rb
|
233
238
|
- spec/shoryuken/middleware/chain_spec.rb
|