shoryuken 5.0.0 → 5.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/FUNDING.yml +12 -0
- data/.rubocop.yml +3 -6
- data/.travis.yml +0 -4
- data/CHANGELOG.md +26 -0
- data/README.md +2 -2
- data/bin/cli/sqs.rb +9 -1
- data/lib/shoryuken.rb +4 -1
- data/lib/shoryuken/default_worker_registry.rb +1 -1
- data/lib/shoryuken/environment_loader.rb +0 -12
- data/lib/shoryuken/extensions/active_job_concurrent_send_adapter.rb +50 -0
- data/lib/shoryuken/options.rb +5 -3
- data/lib/shoryuken/queue.rb +39 -11
- data/lib/shoryuken/version.rb +1 -1
- data/shoryuken.gemspec +1 -1
- data/spec/shared_examples_for_active_job.rb +62 -0
- data/spec/shoryuken/environment_loader_spec.rb +24 -0
- data/spec/shoryuken/extensions/active_job_adapter_spec.rb +2 -59
- data/spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb +35 -0
- data/spec/shoryuken/options_spec.rb +18 -0
- data/spec/shoryuken/queue_spec.rb +23 -0
- metadata +16 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c2a0f01f40da91819e9e8e0b127a2504fccdd927365866f7971a7df207cea287
|
4
|
+
data.tar.gz: 3da85303fc49af5b0ad06888d667bde14ef12b79c595c98cdc3232d085d08350
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8a94cf02d3c7c25bf8282097449b83f1f4f3b4816b0e7be396048585ccc5667e9dcbdfe839f593c0ed04b2304fc1c0ca2d5d9564c86b854de63de6dafddc5de9
|
7
|
+
data.tar.gz: 20844e414f26d4a6a4e9d3db14bd4ea89bad4550486b91a1a1b8ba6eff2145e89fdf63007e356c707e54dbe8e0d1bb608844ec81db02516e84c4971633fd621d
|
data/.github/FUNDING.yml
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
# These are supported funding model platforms
|
2
|
+
|
3
|
+
github: phstc
|
4
|
+
patreon: # Replace with a single Patreon username
|
5
|
+
open_collective: # Replace with a single Open Collective username
|
6
|
+
ko_fi: # Replace with a single Ko-fi username
|
7
|
+
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
|
8
|
+
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
|
9
|
+
liberapay: # Replace with a single Liberapay username
|
10
|
+
issuehunt: # Replace with a single IssueHunt username
|
11
|
+
otechie: # Replace with a single Otechie username
|
12
|
+
custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
|
data/.rubocop.yml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
AllCops:
|
2
2
|
Exclude:
|
3
3
|
- '**/Gemfile'
|
4
|
-
TargetRubyVersion: 2.
|
4
|
+
TargetRubyVersion: 2.4
|
5
5
|
|
6
6
|
Metrics/PerceivedComplexity:
|
7
7
|
Enabled: false
|
@@ -32,7 +32,7 @@ Style/Alias:
|
|
32
32
|
Style/PerlBackrefs:
|
33
33
|
Enabled: false
|
34
34
|
|
35
|
-
Layout/
|
35
|
+
Layout/TrailingEmptyLines:
|
36
36
|
Enabled: false
|
37
37
|
|
38
38
|
# Override the HoundCI custom rules (they do not use Rubocop defaults)
|
@@ -76,7 +76,7 @@ Style/GuardClause:
|
|
76
76
|
Style/RegexpLiteral:
|
77
77
|
Enabled: false
|
78
78
|
|
79
|
-
Lint/
|
79
|
+
Lint/SuppressedException:
|
80
80
|
Enabled: false
|
81
81
|
|
82
82
|
Lint/AssignmentInCondition:
|
@@ -117,6 +117,3 @@ Security/YAMLLoad:
|
|
117
117
|
|
118
118
|
Naming/MemoizedInstanceVariableName:
|
119
119
|
Enabled: false
|
120
|
-
|
121
|
-
Performance/RedundantBlockCall:
|
122
|
-
Enabled: false
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,29 @@
|
|
1
|
+
## [v5.0.5] - 2020-06-07
|
2
|
+
|
3
|
+
- Add ability to configure queue by ARN
|
4
|
+
- [#603](https://github.com/phstc/shoryuken/pull/603)
|
5
|
+
|
6
|
+
## [v5.0.4] - 2020-02-20
|
7
|
+
|
8
|
+
- Add endpoint option to SQS CLI
|
9
|
+
- [#595](https://github.com/phstc/shoryuken/pull/595)
|
10
|
+
|
11
|
+
## [v5.0.3] - 2019-11-30
|
12
|
+
|
13
|
+
- Add support for sending messages asynchronous with Active Job using `shoryuken_concurrent_send`
|
14
|
+
- [#589](https://github.com/phstc/shoryuken/pull/589)
|
15
|
+
- [#588](https://github.com/phstc/shoryuken/pull/588)
|
16
|
+
|
17
|
+
## [v5.0.2] - 2019-11-02
|
18
|
+
|
19
|
+
- Fix Queue order is reversed if passed through CLI
|
20
|
+
- [#571](https://github.com/phstc/shoryuken/pull/583)
|
21
|
+
|
22
|
+
## [v5.0.1] - 2019-06-19
|
23
|
+
|
24
|
+
- Add back attr_accessor for `stop_callback`
|
25
|
+
- [#571](https://github.com/phstc/shoryuken/pull/571)
|
26
|
+
|
1
27
|
## [v5.0.0] - 2019-06-18
|
2
28
|
|
3
29
|
- Fix bug where empty queues were not paused in batch processing mode
|
data/README.md
CHANGED
@@ -21,7 +21,7 @@ Shoryuken _sho-ryu-ken_ is a super-efficient [Amazon SQS](https://aws.amazon.com
|
|
21
21
|
|
22
22
|
## Requirements
|
23
23
|
|
24
|
-
Ruby 2.
|
24
|
+
Ruby 2.4 or greater.
|
25
25
|
|
26
26
|
## Installation
|
27
27
|
|
@@ -37,7 +37,7 @@ If you are using AWS SDK version 3, please also add this line:
|
|
37
37
|
gem 'aws-sdk-sqs'
|
38
38
|
```
|
39
39
|
|
40
|
-
The extra gem `aws-sdk-sqs` is required in order to keep Shoryuken compatible with AWS SDK version 2 and 3.
|
40
|
+
The extra gem `aws-sdk-sqs` is required in order to keep Shoryuken compatible with AWS SDK version 2 and 3.
|
41
41
|
|
42
42
|
And then execute:
|
43
43
|
|
data/bin/cli/sqs.rb
CHANGED
@@ -5,6 +5,7 @@ module Shoryuken
|
|
5
5
|
module CLI
|
6
6
|
class SQS < Base
|
7
7
|
namespace :sqs
|
8
|
+
class_option :endpoint, aliases: '-e', type: :string, default: ENV['SHORYUKEN_SQS_ENDPOINT'], desc: 'Endpoint URL'
|
8
9
|
|
9
10
|
no_commands do
|
10
11
|
def normalize_dump_message(message)
|
@@ -19,8 +20,15 @@ module Shoryuken
|
|
19
20
|
}
|
20
21
|
end
|
21
22
|
|
23
|
+
def client_options
|
24
|
+
endpoint = options[:endpoint]
|
25
|
+
{}.tap do |hash|
|
26
|
+
hash[:endpoint] = endpoint unless endpoint.to_s.empty?
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
22
30
|
def sqs
|
23
|
-
@_sqs ||= Aws::SQS::Client.new
|
31
|
+
@_sqs ||= Aws::SQS::Client.new(client_options)
|
24
32
|
end
|
25
33
|
|
26
34
|
def find_queue_url(queue_name)
|
data/lib/shoryuken.rb
CHANGED
@@ -88,4 +88,7 @@ module Shoryuken
|
|
88
88
|
)
|
89
89
|
end
|
90
90
|
|
91
|
-
|
91
|
+
if Shoryuken.active_job?
|
92
|
+
require 'shoryuken/extensions/active_job_adapter'
|
93
|
+
require 'shoryuken/extensions/active_job_concurrent_send_adapter'
|
94
|
+
end
|
@@ -20,7 +20,6 @@ module Shoryuken
|
|
20
20
|
def setup_options
|
21
21
|
initialize_options
|
22
22
|
initialize_logger
|
23
|
-
merge_cli_defined_queues
|
24
23
|
end
|
25
24
|
|
26
25
|
def load
|
@@ -76,17 +75,6 @@ module Shoryuken
|
|
76
75
|
end
|
77
76
|
end
|
78
77
|
|
79
|
-
def merge_cli_defined_queues
|
80
|
-
cli_defined_queues = options[:queues].to_a
|
81
|
-
|
82
|
-
cli_defined_queues.each do |cli_defined_queue|
|
83
|
-
# CLI defined queues override config_file defined queues
|
84
|
-
Shoryuken.options[:queues].delete_if { |config_file_queue| config_file_queue[0] == cli_defined_queue[0] }
|
85
|
-
|
86
|
-
Shoryuken.options[:queues] << cli_defined_queue
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
78
|
def prefix_active_job_queue_name(queue_name, weight)
|
91
79
|
queue_name_prefix = ::ActiveJob::Base.queue_name_prefix
|
92
80
|
queue_name_delimiter = ::ActiveJob::Base.queue_name_delimiter
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# ActiveJob docs: http://edgeguides.rubyonrails.org/active_job_basics.html
|
2
|
+
# Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
|
3
|
+
module ActiveJob
|
4
|
+
module QueueAdapters
|
5
|
+
# == Shoryuken concurrent adapter for Active Job
|
6
|
+
#
|
7
|
+
# This adapter sends messages asynchronously (ie non-blocking) and allows
|
8
|
+
# the caller to set up handlers for both success and failure
|
9
|
+
#
|
10
|
+
# To use this adapter, set up as:
|
11
|
+
#
|
12
|
+
# success_handler = ->(response, job, options) { StatsD.increment("#{job.class.name}.success") }
|
13
|
+
# error_handler = ->(err, job, options) { StatsD.increment("#{job.class.name}.failure") }
|
14
|
+
#
|
15
|
+
# adapter = ActiveJob::QueueAdapters::ShoryukenConcurrentSendAdapter.new(success_handler, error_handler)
|
16
|
+
#
|
17
|
+
# config.active_job.queue_adapter = adapter
|
18
|
+
class ShoryukenConcurrentSendAdapter < ShoryukenAdapter
|
19
|
+
def initialize(success_handler = nil, error_handler = nil)
|
20
|
+
@success_handler = success_handler
|
21
|
+
@error_handler = error_handler
|
22
|
+
end
|
23
|
+
|
24
|
+
def enqueue(job, options = {})
|
25
|
+
send_concurrently(job, options) { |f_job, f_options| super(f_job, f_options) }
|
26
|
+
end
|
27
|
+
|
28
|
+
def success_handler
|
29
|
+
@success_handler ||= ->(_send_message_response, _job, _options) { nil }
|
30
|
+
end
|
31
|
+
|
32
|
+
def error_handler
|
33
|
+
@error_handler ||= begin
|
34
|
+
lambda { |error, job, _options|
|
35
|
+
Shoryuken.logger.warn("Failed to enqueue job: #{job.inspect} due to error: #{error}")
|
36
|
+
}
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def send_concurrently(job, options)
|
43
|
+
Concurrent::Promises
|
44
|
+
.future(job, options) { |f_job, f_options| [yield(f_job, f_options), f_job, f_options] }
|
45
|
+
.then { |send_message_response, f_job, f_options| success_handler.call(send_message_response, f_job, f_options) }
|
46
|
+
.rescue(job, options) { |err, f_job, f_options| error_handler.call(err, f_job, f_options) }
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
data/lib/shoryuken/options.rb
CHANGED
@@ -14,9 +14,11 @@ module Shoryuken
|
|
14
14
|
}
|
15
15
|
}.freeze
|
16
16
|
|
17
|
-
attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout, :
|
18
|
-
:launcher_executor,
|
19
|
-
:start_callback, :worker_executor, :worker_registry
|
17
|
+
attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout, :groups,
|
18
|
+
:launcher_executor,
|
19
|
+
:start_callback, :stop_callback, :worker_executor, :worker_registry
|
20
|
+
attr_writer :default_worker_options, :sqs_client
|
21
|
+
attr_reader :sqs_client_receive_message_opts
|
20
22
|
|
21
23
|
def initialize
|
22
24
|
self.groups = {}
|
data/lib/shoryuken/queue.rb
CHANGED
@@ -8,9 +8,9 @@ module Shoryuken
|
|
8
8
|
|
9
9
|
attr_accessor :name, :client, :url
|
10
10
|
|
11
|
-
def initialize(client,
|
11
|
+
def initialize(client, name_or_url_or_arn)
|
12
12
|
self.client = client
|
13
|
-
set_name_and_url(
|
13
|
+
set_name_and_url(name_or_url_or_arn)
|
14
14
|
end
|
15
15
|
|
16
16
|
def visibility_timeout
|
@@ -50,32 +50,60 @@ module Shoryuken
|
|
50
50
|
# Make sure the memoization work with boolean to avoid multiple calls to SQS
|
51
51
|
# see https://github.com/phstc/shoryuken/pull/529
|
52
52
|
return @_fifo if defined?(@_fifo)
|
53
|
+
|
53
54
|
@_fifo = queue_attributes.attributes[FIFO_ATTR] == 'true'
|
55
|
+
@_fifo
|
54
56
|
end
|
55
57
|
|
56
58
|
private
|
57
59
|
|
58
|
-
def
|
60
|
+
def initialize_fifo_attribute
|
61
|
+
# calling fifo? will also initialize it
|
62
|
+
fifo?
|
63
|
+
end
|
64
|
+
|
65
|
+
def set_by_name(name) # rubocop:disable Naming/AccessorMethodName
|
59
66
|
self.name = name
|
60
67
|
self.url = client.get_queue_url(queue_name: name).queue_url
|
61
68
|
end
|
62
69
|
|
63
|
-
def set_by_url(url)
|
70
|
+
def set_by_url(url) # rubocop:disable Naming/AccessorMethodName
|
64
71
|
self.name = url.split('/').last
|
65
72
|
self.url = url
|
66
73
|
end
|
67
74
|
|
68
|
-
def
|
69
|
-
|
70
|
-
|
75
|
+
def arn_to_url(arn_str)
|
76
|
+
*, region, account_id, resource = arn_str.split(':')
|
77
|
+
|
78
|
+
required = [region, account_id, resource].map(&:to_s)
|
79
|
+
valid = required.none?(&:empty?)
|
80
|
+
|
81
|
+
abort "Invalid ARN: #{arn_str}. A valid ARN must include: region, account_id and resource." unless valid
|
82
|
+
|
83
|
+
"https://sqs.#{region}.amazonaws.com/#{account_id}/#{resource}"
|
84
|
+
end
|
85
|
+
|
86
|
+
def set_name_and_url(name_or_url_or_arn) # rubocop:disable Naming/AccessorMethodName
|
87
|
+
if name_or_url_or_arn.include?('://')
|
88
|
+
set_by_url(name_or_url_or_arn)
|
89
|
+
|
90
|
+
# anticipate the fifo? checker for validating the queue URL
|
91
|
+
initialize_fifo_attribute
|
92
|
+
return
|
93
|
+
end
|
94
|
+
|
95
|
+
if name_or_url_or_arn.start_with?('arn:')
|
96
|
+
url = arn_to_url(name_or_url_or_arn)
|
97
|
+
set_by_url(url)
|
71
98
|
|
72
99
|
# anticipate the fifo? checker for validating the queue URL
|
73
|
-
|
100
|
+
initialize_fifo_attribute
|
101
|
+
return
|
74
102
|
end
|
75
103
|
|
76
|
-
set_by_name(
|
77
|
-
rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue =>
|
78
|
-
raise
|
104
|
+
set_by_name(name_or_url_or_arn)
|
105
|
+
rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue => e
|
106
|
+
raise e, "The specified queue #{name_or_url_or_arn} does not exist."
|
79
107
|
end
|
80
108
|
|
81
109
|
def queue_attributes
|
data/lib/shoryuken/version.rb
CHANGED
data/shoryuken.gemspec
CHANGED
@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.require_paths = ['lib']
|
19
19
|
|
20
20
|
spec.add_development_dependency 'dotenv'
|
21
|
-
spec.add_development_dependency 'pry-byebug'
|
21
|
+
spec.add_development_dependency 'pry-byebug', '3.9.0'
|
22
22
|
spec.add_development_dependency 'rake'
|
23
23
|
spec.add_development_dependency 'rspec'
|
24
24
|
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# rubocop:disable Metrics/BlockLength
|
2
|
+
RSpec.shared_examples 'active_job_adapters' do
|
3
|
+
let(:job) { double 'Job', id: '123', queue_name: 'queue' }
|
4
|
+
let(:fifo) { false }
|
5
|
+
let(:queue) { double 'Queue', fifo?: fifo }
|
6
|
+
|
7
|
+
before do
|
8
|
+
allow(Shoryuken::Client).to receive(:queues).with(job.queue_name).and_return(queue)
|
9
|
+
allow(job).to receive(:serialize).and_return(
|
10
|
+
'job_class' => 'Worker',
|
11
|
+
'job_id' => job.id,
|
12
|
+
'queue_name' => job.queue_name,
|
13
|
+
'arguments' => nil,
|
14
|
+
'locale' => nil
|
15
|
+
)
|
16
|
+
end
|
17
|
+
|
18
|
+
describe '#enqueue' do
|
19
|
+
specify do
|
20
|
+
expect(queue).to receive(:send_message) do |hash|
|
21
|
+
expect(hash[:message_deduplication_id]).to_not be
|
22
|
+
end
|
23
|
+
expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
|
24
|
+
|
25
|
+
subject.enqueue(job)
|
26
|
+
end
|
27
|
+
|
28
|
+
context 'when fifo' do
|
29
|
+
let(:fifo) { true }
|
30
|
+
|
31
|
+
it 'does not include job_id in the deduplication_id' do
|
32
|
+
expect(queue).to receive(:send_message) do |hash|
|
33
|
+
message_deduplication_id = Digest::SHA256.hexdigest(JSON.dump(job.serialize.except('job_id')))
|
34
|
+
|
35
|
+
expect(hash[:message_deduplication_id]).to eq(message_deduplication_id)
|
36
|
+
end
|
37
|
+
expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
|
38
|
+
|
39
|
+
subject.enqueue(job)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
describe '#enqueue_at' do
|
45
|
+
specify do
|
46
|
+
delay = 1
|
47
|
+
|
48
|
+
expect(queue).to receive(:send_message) do |hash|
|
49
|
+
expect(hash[:message_deduplication_id]).to_not be
|
50
|
+
expect(hash[:delay_seconds]).to eq(delay)
|
51
|
+
end
|
52
|
+
|
53
|
+
expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
|
54
|
+
|
55
|
+
# need to figure out what to require Time.current and N.minutes to remove the stub
|
56
|
+
allow(subject).to receive(:calculate_delay).and_return(delay)
|
57
|
+
|
58
|
+
subject.enqueue_at(job, nil)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
# rubocop:enable Metrics/BlockLength
|
@@ -74,4 +74,28 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
74
74
|
expect(Shoryuken.groups['group1'][:queues]).to eq(%w[test_group1_queue1 test_group1_queue2])
|
75
75
|
end
|
76
76
|
end
|
77
|
+
describe "#setup_options" do
|
78
|
+
let (:cli_queues) { { "queue1"=> 10, "queue2" => 20 } }
|
79
|
+
let (:config_queues) { [["queue1", 8], ["queue2", 4]] }
|
80
|
+
context "when given queues through config and CLI" do
|
81
|
+
specify do
|
82
|
+
allow_any_instance_of(Shoryuken::EnvironmentLoader).to receive(:config_file_options).and_return({ queues: config_queues })
|
83
|
+
Shoryuken::EnvironmentLoader.setup_options(queues: cli_queues)
|
84
|
+
expect(Shoryuken.options[:queues]).to eq(cli_queues)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
context "when given queues through config only" do
|
88
|
+
specify do
|
89
|
+
allow_any_instance_of(Shoryuken::EnvironmentLoader).to receive(:config_file_options).and_return({ queues: config_queues })
|
90
|
+
Shoryuken::EnvironmentLoader.setup_options({})
|
91
|
+
expect(Shoryuken.options[:queues]).to eq(config_queues)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
context "when given queues through CLI only" do
|
95
|
+
specify do
|
96
|
+
Shoryuken::EnvironmentLoader.setup_options(queues: cli_queues)
|
97
|
+
expect(Shoryuken.options[:queues]).to eq(cli_queues)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
77
101
|
end
|
@@ -1,64 +1,7 @@
|
|
1
1
|
require 'spec_helper'
|
2
|
-
require 'active_job'
|
3
2
|
require 'shoryuken/extensions/active_job_adapter'
|
3
|
+
require 'shared_examples_for_active_job'
|
4
4
|
|
5
5
|
RSpec.describe ActiveJob::QueueAdapters::ShoryukenAdapter do
|
6
|
-
|
7
|
-
let(:fifo) { false }
|
8
|
-
let(:queue) { double 'Queue', fifo?: fifo }
|
9
|
-
|
10
|
-
before do
|
11
|
-
allow(Shoryuken::Client).to receive(:queues).with(job.queue_name).and_return(queue)
|
12
|
-
allow(job).to receive(:serialize).and_return(
|
13
|
-
'job_class' => 'Worker',
|
14
|
-
'job_id' => job.id,
|
15
|
-
'queue_name' => job.queue_name,
|
16
|
-
'arguments' => nil,
|
17
|
-
'locale' => nil
|
18
|
-
)
|
19
|
-
end
|
20
|
-
|
21
|
-
describe '#enqueue' do
|
22
|
-
specify do
|
23
|
-
expect(queue).to receive(:send_message) do |hash|
|
24
|
-
expect(hash[:message_deduplication_id]).to_not be
|
25
|
-
end
|
26
|
-
expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
|
27
|
-
|
28
|
-
subject.enqueue(job)
|
29
|
-
end
|
30
|
-
|
31
|
-
context 'when fifo' do
|
32
|
-
let(:fifo) { true }
|
33
|
-
|
34
|
-
it 'does not include job_id in the deduplication_id' do
|
35
|
-
expect(queue).to receive(:send_message) do |hash|
|
36
|
-
message_deduplication_id = Digest::SHA256.hexdigest(JSON.dump(job.serialize.except('job_id')))
|
37
|
-
|
38
|
-
expect(hash[:message_deduplication_id]).to eq(message_deduplication_id)
|
39
|
-
end
|
40
|
-
expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
|
41
|
-
|
42
|
-
subject.enqueue(job)
|
43
|
-
end
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
describe '#enqueue_at' do
|
48
|
-
specify do
|
49
|
-
delay = 1
|
50
|
-
|
51
|
-
expect(queue).to receive(:send_message) do |hash|
|
52
|
-
expect(hash[:message_deduplication_id]).to_not be
|
53
|
-
expect(hash[:delay_seconds]).to eq(delay)
|
54
|
-
end
|
55
|
-
|
56
|
-
expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
|
57
|
-
|
58
|
-
# need to figure out what to require Time.current and N.minutes to remove the stub
|
59
|
-
allow(subject).to receive(:calculate_delay).and_return(delay)
|
60
|
-
|
61
|
-
subject.enqueue_at(job, nil)
|
62
|
-
end
|
63
|
-
end
|
6
|
+
include_examples 'active_job_adapters'
|
64
7
|
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'shared_examples_for_active_job'
|
3
|
+
require 'shoryuken/extensions/active_job_adapter'
|
4
|
+
require 'shoryuken/extensions/active_job_concurrent_send_adapter'
|
5
|
+
|
6
|
+
RSpec.describe ActiveJob::QueueAdapters::ShoryukenConcurrentSendAdapter do
|
7
|
+
include_examples 'active_job_adapters'
|
8
|
+
|
9
|
+
let(:options) { {} }
|
10
|
+
let(:error_handler) { -> {} }
|
11
|
+
let(:success_handler) { -> {} }
|
12
|
+
|
13
|
+
subject { described_class.new(success_handler, error_handler) }
|
14
|
+
|
15
|
+
context 'when success' do
|
16
|
+
it 'calls success_handler' do
|
17
|
+
response = true
|
18
|
+
allow(queue).to receive(:send_message).and_return(response)
|
19
|
+
expect(success_handler).to receive(:call).with(response, job, options)
|
20
|
+
|
21
|
+
subject.enqueue(job, options)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
context 'when failure' do
|
26
|
+
it 'calls error_handler' do
|
27
|
+
response = Aws::SQS::Errors::InternalError.new('error', 'error')
|
28
|
+
|
29
|
+
allow(queue).to receive(:send_message).and_raise(response)
|
30
|
+
expect(error_handler).to receive(:call).with(response, job, options).and_call_original
|
31
|
+
|
32
|
+
subject.enqueue(job, options)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -3,6 +3,24 @@ require 'spec_helper'
|
|
3
3
|
RSpec.describe Shoryuken::Options do
|
4
4
|
subject { Shoryuken.shoryuken_options }
|
5
5
|
|
6
|
+
describe '.on_stop' do
|
7
|
+
specify do
|
8
|
+
on_stop = Proc.new {}
|
9
|
+
Shoryuken.on_stop(&on_stop)
|
10
|
+
|
11
|
+
expect(Shoryuken.stop_callback).to eq(on_stop)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
describe '.on_start' do
|
16
|
+
specify do
|
17
|
+
on_start = Proc.new {}
|
18
|
+
Shoryuken.on_start(&on_start)
|
19
|
+
|
20
|
+
expect(Shoryuken.start_callback).to eq(on_start)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
6
24
|
describe '.add_group adds queues and optional delay' do
|
7
25
|
before do
|
8
26
|
Shoryuken.groups.clear
|
@@ -39,6 +39,29 @@ RSpec.describe Shoryuken::Queue do
|
|
39
39
|
end
|
40
40
|
end
|
41
41
|
|
42
|
+
context 'when queue ARN supplied' do
|
43
|
+
let(:queue_arn) { 'arn:aws:sqs:ap-southeast-2:000000000000:queue-name' }
|
44
|
+
|
45
|
+
it 'initializes by URL and validate the URL' do
|
46
|
+
subject = described_class.new(sqs, queue_arn)
|
47
|
+
|
48
|
+
expect(subject.name).to eq('queue-name')
|
49
|
+
expect(subject.url).to eq('https://sqs.ap-southeast-2.amazonaws.com/000000000000/queue-name')
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
context 'when invalid queue ARN supplied' do
|
54
|
+
let(:queue_arn) { 'arn:aws:sqs::000000000000:queue-name' }
|
55
|
+
|
56
|
+
it 'raises an error' do
|
57
|
+
expect do
|
58
|
+
described_class.new(sqs, queue_arn)
|
59
|
+
end.to raise_error(
|
60
|
+
'Invalid ARN: arn:aws:sqs::000000000000:queue-name. A valid ARN must include: region, account_id and resource.'
|
61
|
+
)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
42
65
|
context 'when queue name supplied' do
|
43
66
|
subject { described_class.new(sqs, queue_name) }
|
44
67
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shoryuken
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.0.
|
4
|
+
version: 5.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pablo Cantero
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-10-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: dotenv
|
@@ -28,16 +28,16 @@ dependencies:
|
|
28
28
|
name: pry-byebug
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - '='
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version:
|
33
|
+
version: 3.9.0
|
34
34
|
type: :development
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - '='
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version:
|
40
|
+
version: 3.9.0
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: rake
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -117,6 +117,7 @@ extensions: []
|
|
117
117
|
extra_rdoc_files: []
|
118
118
|
files:
|
119
119
|
- ".codeclimate.yml"
|
120
|
+
- ".github/FUNDING.yml"
|
120
121
|
- ".gitignore"
|
121
122
|
- ".rspec"
|
122
123
|
- ".rubocop.yml"
|
@@ -139,6 +140,7 @@ files:
|
|
139
140
|
- lib/shoryuken/default_worker_registry.rb
|
140
141
|
- lib/shoryuken/environment_loader.rb
|
141
142
|
- lib/shoryuken/extensions/active_job_adapter.rb
|
143
|
+
- lib/shoryuken/extensions/active_job_concurrent_send_adapter.rb
|
142
144
|
- lib/shoryuken/fetcher.rb
|
143
145
|
- lib/shoryuken/launcher.rb
|
144
146
|
- lib/shoryuken/logging.rb
|
@@ -166,6 +168,7 @@ files:
|
|
166
168
|
- shoryuken.gemspec
|
167
169
|
- shoryuken.jpg
|
168
170
|
- spec/integration/launcher_spec.rb
|
171
|
+
- spec/shared_examples_for_active_job.rb
|
169
172
|
- spec/shoryuken.yml
|
170
173
|
- spec/shoryuken/body_parser_spec.rb
|
171
174
|
- spec/shoryuken/client_spec.rb
|
@@ -173,6 +176,7 @@ files:
|
|
173
176
|
- spec/shoryuken/default_worker_registry_spec.rb
|
174
177
|
- spec/shoryuken/environment_loader_spec.rb
|
175
178
|
- spec/shoryuken/extensions/active_job_adapter_spec.rb
|
179
|
+
- spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb
|
176
180
|
- spec/shoryuken/fetcher_spec.rb
|
177
181
|
- spec/shoryuken/manager_spec.rb
|
178
182
|
- spec/shoryuken/middleware/chain_spec.rb
|
@@ -198,7 +202,7 @@ homepage: https://github.com/phstc/shoryuken
|
|
198
202
|
licenses:
|
199
203
|
- LGPL-3.0
|
200
204
|
metadata: {}
|
201
|
-
post_install_message:
|
205
|
+
post_install_message:
|
202
206
|
rdoc_options: []
|
203
207
|
require_paths:
|
204
208
|
- lib
|
@@ -213,13 +217,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
213
217
|
- !ruby/object:Gem::Version
|
214
218
|
version: '0'
|
215
219
|
requirements: []
|
216
|
-
rubyforge_project:
|
220
|
+
rubyforge_project:
|
217
221
|
rubygems_version: 2.7.6.2
|
218
|
-
signing_key:
|
222
|
+
signing_key:
|
219
223
|
specification_version: 4
|
220
224
|
summary: Shoryuken is a super efficient AWS SQS thread based message processor
|
221
225
|
test_files:
|
222
226
|
- spec/integration/launcher_spec.rb
|
227
|
+
- spec/shared_examples_for_active_job.rb
|
223
228
|
- spec/shoryuken.yml
|
224
229
|
- spec/shoryuken/body_parser_spec.rb
|
225
230
|
- spec/shoryuken/client_spec.rb
|
@@ -227,6 +232,7 @@ test_files:
|
|
227
232
|
- spec/shoryuken/default_worker_registry_spec.rb
|
228
233
|
- spec/shoryuken/environment_loader_spec.rb
|
229
234
|
- spec/shoryuken/extensions/active_job_adapter_spec.rb
|
235
|
+
- spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb
|
230
236
|
- spec/shoryuken/fetcher_spec.rb
|
231
237
|
- spec/shoryuken/manager_spec.rb
|
232
238
|
- spec/shoryuken/middleware/chain_spec.rb
|