shoryuken 5.0.2 → 5.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/specs.yml +57 -0
- data/.reek.yml +5 -0
- data/.rubocop.yml +3 -6
- data/Appraisals +28 -0
- data/CHANGELOG.md +39 -0
- data/Gemfile +3 -1
- data/README.md +19 -3
- data/Rakefile +3 -0
- data/bin/cli/sqs.rb +9 -1
- data/gemfiles/.gitignore +1 -0
- data/gemfiles/rails_4_2.gemfile +20 -0
- data/gemfiles/rails_5_2.gemfile +21 -0
- data/gemfiles/rails_6_0.gemfile +21 -0
- data/gemfiles/rails_6_1.gemfile +21 -0
- data/lib/shoryuken.rb +5 -1
- data/lib/shoryuken/default_worker_registry.rb +1 -1
- data/lib/shoryuken/environment_loader.rb +5 -1
- data/lib/shoryuken/extensions/active_job_adapter.rb +21 -16
- data/lib/shoryuken/extensions/active_job_concurrent_send_adapter.rb +50 -0
- data/lib/shoryuken/extensions/active_job_extensions.rb +37 -0
- data/lib/shoryuken/manager.rb +10 -4
- data/lib/shoryuken/polling/base.rb +2 -0
- data/lib/shoryuken/polling/strict_priority.rb +6 -0
- data/lib/shoryuken/polling/weighted_round_robin.rb +11 -0
- data/lib/shoryuken/queue.rb +39 -11
- data/lib/shoryuken/version.rb +1 -1
- data/shoryuken.gemspec +0 -1
- data/spec/shared_examples_for_active_job.rb +279 -0
- data/spec/shoryuken/extensions/active_job_adapter_spec.rb +2 -59
- data/spec/shoryuken/extensions/active_job_base_spec.rb +73 -0
- data/spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb +35 -0
- data/spec/shoryuken/manager_spec.rb +24 -0
- data/spec/shoryuken/polling/strict_priority_spec.rb +10 -0
- data/spec/shoryuken/polling/weighted_round_robin_spec.rb +10 -0
- data/spec/shoryuken/queue_spec.rb +23 -0
- data/spec/spec_helper.rb +6 -6
- metadata +22 -22
- data/.travis.yml +0 -34
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 313fe407e7c7715e8a83bd3fcef713cb5755d31e9d94345fe26157c823f4b156
|
4
|
+
data.tar.gz: 53890b5dc5dcf0d12fd5153b459d9cb317dace747d95516f754bd320be7eb105
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b349de0985d3beac244ed9f1b8886c7b71e18a810936d1cf1e89baddff507ee5b83af100a598653976ba8e337f0e8ad25f3db6b178b6bbd544e9696772b63160
|
7
|
+
data.tar.gz: 92ef7d6d5648485c1556225a81e6808d6ba127342e3edcb93cacb10e4f1070077f40f6b5ec3b9a1809226e092eb99c2710c2d238c4f16ea3072455f83c2eb643
|
@@ -0,0 +1,57 @@
|
|
1
|
+
name: Specs
|
2
|
+
|
3
|
+
on:
|
4
|
+
- push
|
5
|
+
- pull_request
|
6
|
+
|
7
|
+
jobs:
|
8
|
+
all_specs:
|
9
|
+
name: All Specs
|
10
|
+
strategy:
|
11
|
+
matrix:
|
12
|
+
ruby: ['2.4.4', '2.5.1', '2.6.3']
|
13
|
+
gemfile: ['Gemfile', 'Gemfile.aws-sdk-core-v2']
|
14
|
+
runs-on: ubuntu-20.04
|
15
|
+
env:
|
16
|
+
BUNDLE_GEMFILE: ${{ matrix.gemfile }}
|
17
|
+
steps:
|
18
|
+
- name: Checkout code
|
19
|
+
uses: actions/checkout@v2
|
20
|
+
- uses: ruby/setup-ruby@v1
|
21
|
+
with:
|
22
|
+
ruby-version: ${{ matrix.ruby }}
|
23
|
+
bundler-cache: true
|
24
|
+
- name: Run specs
|
25
|
+
env:
|
26
|
+
SPEC_ALL: true
|
27
|
+
run: bundle exec rake spec
|
28
|
+
rails_specs:
|
29
|
+
name: Rails Specs
|
30
|
+
strategy:
|
31
|
+
matrix:
|
32
|
+
rails: ['4.2', '5.2', '6.0', '6.1']
|
33
|
+
include:
|
34
|
+
- rails: '4.2'
|
35
|
+
ruby: '2.2'
|
36
|
+
gemfile: gemfiles/rails_4_2.gemfile
|
37
|
+
- rails: '5.2'
|
38
|
+
ruby: '2.5'
|
39
|
+
gemfile: gemfiles/rails_5_2.gemfile
|
40
|
+
- rails: '6.0'
|
41
|
+
ruby: '2.6'
|
42
|
+
gemfile: gemfiles/rails_6_0.gemfile
|
43
|
+
- rails: '6.1'
|
44
|
+
ruby: '3.0'
|
45
|
+
gemfile: gemfiles/rails_6_1.gemfile
|
46
|
+
runs-on: ubuntu-20.04
|
47
|
+
env:
|
48
|
+
BUNDLE_GEMFILE: ${{ matrix.gemfile }}
|
49
|
+
steps:
|
50
|
+
- name: Checkout code
|
51
|
+
uses: actions/checkout@v2
|
52
|
+
- uses: ruby/setup-ruby@v1
|
53
|
+
with:
|
54
|
+
ruby-version: ${{ matrix.ruby }}
|
55
|
+
bundler-cache: true
|
56
|
+
- name: Run Rails specs
|
57
|
+
run: bundle exec rake rails_specs
|
data/.reek.yml
ADDED
data/.rubocop.yml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
AllCops:
|
2
2
|
Exclude:
|
3
3
|
- '**/Gemfile'
|
4
|
-
TargetRubyVersion: 2.
|
4
|
+
TargetRubyVersion: 2.4
|
5
5
|
|
6
6
|
Metrics/PerceivedComplexity:
|
7
7
|
Enabled: false
|
@@ -32,7 +32,7 @@ Style/Alias:
|
|
32
32
|
Style/PerlBackrefs:
|
33
33
|
Enabled: false
|
34
34
|
|
35
|
-
Layout/
|
35
|
+
Layout/TrailingEmptyLines:
|
36
36
|
Enabled: false
|
37
37
|
|
38
38
|
# Override the HoundCI custom rules (they do not use Rubocop defaults)
|
@@ -76,7 +76,7 @@ Style/GuardClause:
|
|
76
76
|
Style/RegexpLiteral:
|
77
77
|
Enabled: false
|
78
78
|
|
79
|
-
Lint/
|
79
|
+
Lint/SuppressedException:
|
80
80
|
Enabled: false
|
81
81
|
|
82
82
|
Lint/AssignmentInCondition:
|
@@ -117,6 +117,3 @@ Security/YAMLLoad:
|
|
117
117
|
|
118
118
|
Naming/MemoizedInstanceVariableName:
|
119
119
|
Enabled: false
|
120
|
-
|
121
|
-
Performance/RedundantBlockCall:
|
122
|
-
Enabled: false
|
data/Appraisals
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
appraise 'rails_4_2' do
|
2
|
+
group :test do
|
3
|
+
gem 'activejob', '~> 4.2'
|
4
|
+
end
|
5
|
+
|
6
|
+
group :development do
|
7
|
+
gem 'appraisal', '~> 2.2'
|
8
|
+
remove_gem 'pry-byebug'
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
appraise 'rails_5_2' do
|
13
|
+
group :test do
|
14
|
+
gem 'activejob', '~> 5.2'
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
appraise 'rails_6_0' do
|
19
|
+
group :test do
|
20
|
+
gem 'activejob', '~> 6.0'
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
appraise 'rails_6_1' do
|
25
|
+
group :test do
|
26
|
+
gem 'activejob', '~> 6.1'
|
27
|
+
end
|
28
|
+
end
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,42 @@
|
|
1
|
+
## [v5.1.0] - 2021-02-06
|
2
|
+
|
3
|
+
- Add support for specifying SQS SendMessage parameters with ActiveJob `.set`
|
4
|
+
- [#635](https://github.com/phstc/shoryuken/pull/635)
|
5
|
+
- [#648](https://github.com/phstc/shoryuken/pull/648)
|
6
|
+
- [#651](https://github.com/phstc/shoryuken/pull/651)
|
7
|
+
|
8
|
+
- Unpause FIFO queues on worker completion
|
9
|
+
- [#644](https://github.com/phstc/shoryuken/pull/644)
|
10
|
+
|
11
|
+
- Add multiple versions of Rails to test matrix
|
12
|
+
- [#647](https://github.com/phstc/shoryuken/pull/647)
|
13
|
+
|
14
|
+
- Migrate from Travis CI to Github Actions
|
15
|
+
- [#649](https://github.com/phstc/shoryuken/pull/649)
|
16
|
+
- [#650](https://github.com/phstc/shoryuken/pull/650)
|
17
|
+
- [#652](https://github.com/phstc/shoryuken/pull/652)
|
18
|
+
|
19
|
+
## [v5.0.6] - 2020-12-30
|
20
|
+
|
21
|
+
- Load ShoryukenConcurrentSendAdapter when loading Rails
|
22
|
+
- [#642](https://github.com/phstc/shoryuken/pull/642)
|
23
|
+
|
24
|
+
## [v5.0.5] - 2020-06-07
|
25
|
+
|
26
|
+
- Add ability to configure queue by ARN
|
27
|
+
- [#603](https://github.com/phstc/shoryuken/pull/603)
|
28
|
+
|
29
|
+
## [v5.0.4] - 2020-02-20
|
30
|
+
|
31
|
+
- Add endpoint option to SQS CLI
|
32
|
+
- [#595](https://github.com/phstc/shoryuken/pull/595)
|
33
|
+
|
34
|
+
## [v5.0.3] - 2019-11-30
|
35
|
+
|
36
|
+
- Add support for sending messages asynchronous with Active Job using `shoryuken_concurrent_send`
|
37
|
+
- [#589](https://github.com/phstc/shoryuken/pull/589)
|
38
|
+
- [#588](https://github.com/phstc/shoryuken/pull/588)
|
39
|
+
|
1
40
|
## [v5.0.2] - 2019-11-02
|
2
41
|
|
3
42
|
- Fix Queue order is reversed if passed through CLI
|
data/Gemfile
CHANGED
@@ -4,7 +4,7 @@ source 'https://rubygems.org'
|
|
4
4
|
gemspec
|
5
5
|
|
6
6
|
group :test do
|
7
|
-
gem 'activejob'
|
7
|
+
gem 'activejob'
|
8
8
|
gem 'aws-sdk-core', '~> 3'
|
9
9
|
gem 'aws-sdk-sqs'
|
10
10
|
gem 'codeclimate-test-reporter', require: nil
|
@@ -14,5 +14,7 @@ group :test do
|
|
14
14
|
end
|
15
15
|
|
16
16
|
group :development do
|
17
|
+
gem 'appraisal', git: 'https://github.com/thoughtbot/appraisal.git'
|
18
|
+
gem 'pry-byebug', '3.9.0'
|
17
19
|
gem 'rubocop'
|
18
20
|
end
|
data/README.md
CHANGED
@@ -1,10 +1,12 @@
|
|
1
|
+
**I'm looking for Shoryuken maintainers, are you interested on helping to maintain Shoryuken? Fill up this form https://forms.gle/8kTso8ixa9Sfp6rJ9**
|
2
|
+
|
1
3
|
# Shoryuken
|
2
4
|
|
3
5
|
![Shoryuken](shoryuken.jpg)
|
4
6
|
|
5
7
|
Shoryuken _sho-ryu-ken_ is a super-efficient [Amazon SQS](https://aws.amazon.com/sqs/) thread-based message processor.
|
6
8
|
|
7
|
-
[![Build Status](https://
|
9
|
+
[![Build Status](https://github.com/ruby-shoryuken/shoryuken/workflows/Specs/badge.svg)](https://github.com/ruby-shoryuken/shoryuken/actions)
|
8
10
|
[![Code Climate](https://codeclimate.com/github/phstc/shoryuken/badges/gpa.svg)](https://codeclimate.com/github/phstc/shoryuken)
|
9
11
|
|
10
12
|
## Key features
|
@@ -21,7 +23,7 @@ Shoryuken _sho-ryu-ken_ is a super-efficient [Amazon SQS](https://aws.amazon.com
|
|
21
23
|
|
22
24
|
## Requirements
|
23
25
|
|
24
|
-
Ruby 2.
|
26
|
+
Ruby 2.4 or greater.
|
25
27
|
|
26
28
|
## Installation
|
27
29
|
|
@@ -37,7 +39,7 @@ If you are using AWS SDK version 3, please also add this line:
|
|
37
39
|
gem 'aws-sdk-sqs'
|
38
40
|
```
|
39
41
|
|
40
|
-
The extra gem `aws-sdk-sqs` is required in order to keep Shoryuken compatible with AWS SDK version 2 and 3.
|
42
|
+
The extra gem `aws-sdk-sqs` is required in order to keep Shoryuken compatible with AWS SDK version 2 and 3.
|
41
43
|
|
42
44
|
And then execute:
|
43
45
|
|
@@ -64,3 +66,17 @@ For more information check the [wiki page](https://github.com/phstc/shoryuken/wi
|
|
64
66
|
3. Commit your changes (`git commit -am 'Add some feature'`)
|
65
67
|
4. Push to the branch (`git push origin my-new-feature`)
|
66
68
|
5. Create a new Pull Request
|
69
|
+
|
70
|
+
### Testing
|
71
|
+
|
72
|
+
To run all specs against the latest dependency vesions, execute
|
73
|
+
|
74
|
+
```sh
|
75
|
+
bundle exec rake spec
|
76
|
+
```
|
77
|
+
|
78
|
+
To run all Rails-related specs against all supported versions of Rails, execute
|
79
|
+
|
80
|
+
```sh
|
81
|
+
bundle exec appraisal rake rails_specs
|
82
|
+
```
|
data/Rakefile
CHANGED
@@ -4,6 +4,9 @@ $stdout.sync = true
|
|
4
4
|
begin
|
5
5
|
require 'rspec/core/rake_task'
|
6
6
|
RSpec::Core::RakeTask.new(:spec)
|
7
|
+
|
8
|
+
rails_task = RSpec::Core::RakeTask.new(:rails_specs)
|
9
|
+
rails_task.pattern = 'spec/shoryuken/{environment_loader_spec,extensions/active_job_*}.rb'
|
7
10
|
rescue LoadError
|
8
11
|
end
|
9
12
|
|
data/bin/cli/sqs.rb
CHANGED
@@ -5,6 +5,7 @@ module Shoryuken
|
|
5
5
|
module CLI
|
6
6
|
class SQS < Base
|
7
7
|
namespace :sqs
|
8
|
+
class_option :endpoint, aliases: '-e', type: :string, default: ENV['SHORYUKEN_SQS_ENDPOINT'], desc: 'Endpoint URL'
|
8
9
|
|
9
10
|
no_commands do
|
10
11
|
def normalize_dump_message(message)
|
@@ -19,8 +20,15 @@ module Shoryuken
|
|
19
20
|
}
|
20
21
|
end
|
21
22
|
|
23
|
+
def client_options
|
24
|
+
endpoint = options[:endpoint]
|
25
|
+
{}.tap do |hash|
|
26
|
+
hash[:endpoint] = endpoint unless endpoint.to_s.empty?
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
22
30
|
def sqs
|
23
|
-
@_sqs ||= Aws::SQS::Client.new
|
31
|
+
@_sqs ||= Aws::SQS::Client.new(client_options)
|
24
32
|
end
|
25
33
|
|
26
34
|
def find_queue_url(queue_name)
|
data/gemfiles/.gitignore
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
*.gemfile.lock
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# This file was generated by Appraisal
|
2
|
+
|
3
|
+
source "https://rubygems.org"
|
4
|
+
|
5
|
+
group :test do
|
6
|
+
gem "activejob", "~> 4.2"
|
7
|
+
gem "aws-sdk-core", "~> 3"
|
8
|
+
gem "aws-sdk-sqs"
|
9
|
+
gem "codeclimate-test-reporter", require: nil
|
10
|
+
gem "httparty"
|
11
|
+
gem "multi_xml"
|
12
|
+
gem "simplecov"
|
13
|
+
end
|
14
|
+
|
15
|
+
group :development do
|
16
|
+
gem "appraisal", "~> 2.2"
|
17
|
+
gem "rubocop"
|
18
|
+
end
|
19
|
+
|
20
|
+
gemspec path: "../"
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# This file was generated by Appraisal
|
2
|
+
|
3
|
+
source "https://rubygems.org"
|
4
|
+
|
5
|
+
group :test do
|
6
|
+
gem "activejob", "~> 5.2"
|
7
|
+
gem "aws-sdk-core", "~> 3"
|
8
|
+
gem "aws-sdk-sqs"
|
9
|
+
gem "codeclimate-test-reporter", require: nil
|
10
|
+
gem "httparty"
|
11
|
+
gem "multi_xml"
|
12
|
+
gem "simplecov"
|
13
|
+
end
|
14
|
+
|
15
|
+
group :development do
|
16
|
+
gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
|
17
|
+
gem "pry-byebug", "3.9.0"
|
18
|
+
gem "rubocop"
|
19
|
+
end
|
20
|
+
|
21
|
+
gemspec path: "../"
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# This file was generated by Appraisal
|
2
|
+
|
3
|
+
source "https://rubygems.org"
|
4
|
+
|
5
|
+
group :test do
|
6
|
+
gem "activejob", "~> 6.0"
|
7
|
+
gem "aws-sdk-core", "~> 3"
|
8
|
+
gem "aws-sdk-sqs"
|
9
|
+
gem "codeclimate-test-reporter", require: nil
|
10
|
+
gem "httparty"
|
11
|
+
gem "multi_xml"
|
12
|
+
gem "simplecov"
|
13
|
+
end
|
14
|
+
|
15
|
+
group :development do
|
16
|
+
gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
|
17
|
+
gem "pry-byebug", "3.9.0"
|
18
|
+
gem "rubocop"
|
19
|
+
end
|
20
|
+
|
21
|
+
gemspec path: "../"
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# This file was generated by Appraisal
|
2
|
+
|
3
|
+
source "https://rubygems.org"
|
4
|
+
|
5
|
+
group :test do
|
6
|
+
gem "activejob", "~> 6.1"
|
7
|
+
gem "aws-sdk-core", "~> 3"
|
8
|
+
gem "aws-sdk-sqs"
|
9
|
+
gem "codeclimate-test-reporter", require: nil
|
10
|
+
gem "httparty"
|
11
|
+
gem "multi_xml"
|
12
|
+
gem "simplecov"
|
13
|
+
end
|
14
|
+
|
15
|
+
group :development do
|
16
|
+
gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
|
17
|
+
gem "pry-byebug", "3.9.0"
|
18
|
+
gem "rubocop"
|
19
|
+
end
|
20
|
+
|
21
|
+
gemspec path: "../"
|
data/lib/shoryuken.rb
CHANGED
@@ -88,4 +88,8 @@ module Shoryuken
|
|
88
88
|
)
|
89
89
|
end
|
90
90
|
|
91
|
-
|
91
|
+
if Shoryuken.active_job?
|
92
|
+
require 'shoryuken/extensions/active_job_extensions'
|
93
|
+
require 'shoryuken/extensions/active_job_adapter'
|
94
|
+
require 'shoryuken/extensions/active_job_concurrent_send_adapter'
|
95
|
+
end
|
@@ -70,7 +70,11 @@ module Shoryuken
|
|
70
70
|
::Rails.application.config.eager_load = true
|
71
71
|
end
|
72
72
|
end
|
73
|
-
|
73
|
+
if Shoryuken.active_job?
|
74
|
+
require 'shoryuken/extensions/active_job_extensions'
|
75
|
+
require 'shoryuken/extensions/active_job_adapter'
|
76
|
+
require 'shoryuken/extensions/active_job_concurrent_send_adapter'
|
77
|
+
end
|
74
78
|
require File.expand_path('config/environment.rb')
|
75
79
|
end
|
76
80
|
end
|
@@ -33,8 +33,12 @@ module ActiveJob
|
|
33
33
|
def enqueue(job, options = {}) #:nodoc:
|
34
34
|
register_worker!(job)
|
35
35
|
|
36
|
+
job.sqs_send_message_parameters.merge! options
|
37
|
+
|
36
38
|
queue = Shoryuken::Client.queues(job.queue_name)
|
37
|
-
|
39
|
+
send_message_params = message queue, job
|
40
|
+
job.sqs_send_message_parameters = send_message_params
|
41
|
+
queue.send_message send_message_params
|
38
42
|
end
|
39
43
|
|
40
44
|
def enqueue_at(job, timestamp) #:nodoc:
|
@@ -50,35 +54,29 @@ module ActiveJob
|
|
50
54
|
delay
|
51
55
|
end
|
52
56
|
|
53
|
-
def message(queue, job
|
57
|
+
def message(queue, job)
|
54
58
|
body = job.serialize
|
59
|
+
job_params = job.sqs_send_message_parameters
|
60
|
+
|
61
|
+
attributes = job_params[:message_attributes] || {}
|
55
62
|
|
56
|
-
msg = {
|
63
|
+
msg = {
|
64
|
+
message_body: body,
|
65
|
+
message_attributes: attributes.merge(MESSAGE_ATTRIBUTES)
|
66
|
+
}
|
57
67
|
|
58
68
|
if queue.fifo?
|
59
69
|
# See https://github.com/phstc/shoryuken/issues/457
|
60
70
|
msg[:message_deduplication_id] = Digest::SHA256.hexdigest(JSON.dump(body.except('job_id')))
|
61
71
|
end
|
62
72
|
|
63
|
-
msg
|
64
|
-
msg[:message_attributes] = message_attributes
|
65
|
-
|
66
|
-
msg.merge(options)
|
73
|
+
msg.merge(job_params.except(:message_attributes))
|
67
74
|
end
|
68
75
|
|
69
76
|
def register_worker!(job)
|
70
77
|
Shoryuken.register_worker(job.queue_name, JobWrapper)
|
71
78
|
end
|
72
79
|
|
73
|
-
def message_attributes
|
74
|
-
@message_attributes ||= {
|
75
|
-
'shoryuken_class' => {
|
76
|
-
string_value: JobWrapper.to_s,
|
77
|
-
data_type: 'String'
|
78
|
-
}
|
79
|
-
}
|
80
|
-
end
|
81
|
-
|
82
80
|
class JobWrapper #:nodoc:
|
83
81
|
include Shoryuken::Worker
|
84
82
|
|
@@ -88,6 +86,13 @@ module ActiveJob
|
|
88
86
|
Base.execute hash
|
89
87
|
end
|
90
88
|
end
|
89
|
+
|
90
|
+
MESSAGE_ATTRIBUTES = {
|
91
|
+
'shoryuken_class' => {
|
92
|
+
string_value: JobWrapper.to_s,
|
93
|
+
data_type: 'String'
|
94
|
+
}
|
95
|
+
}.freeze
|
91
96
|
end
|
92
97
|
end
|
93
98
|
end
|