shoryuken 5.2.2 → 5.3.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/specs.yml +2 -2
- data/.gitignore +1 -1
- data/.rubocop.yml +1 -1
- data/Appraisals +8 -0
- data/CHANGELOG.md +26 -0
- data/Gemfile +1 -1
- data/gemfiles/aws_sdk_core_2.gemfile +21 -0
- data/lib/shoryuken/environment_loader.rb +9 -1
- data/lib/shoryuken/launcher.rb +19 -0
- data/lib/shoryuken/manager.rb +40 -10
- data/lib/shoryuken/message.rb +11 -28
- data/lib/shoryuken/options.rb +1 -0
- data/lib/shoryuken/polling/strict_priority.rb +4 -2
- data/lib/shoryuken/polling/weighted_round_robin.rb +3 -5
- data/lib/shoryuken/runner.rb +4 -0
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken.rb +4 -0
- data/spec/shoryuken/environment_loader_spec.rb +39 -6
- data/spec/shoryuken/launcher_spec.rb +60 -0
- data/spec/shoryuken/manager_spec.rb +33 -1
- data/spec/shoryuken/polling/weighted_round_robin_spec.rb +31 -6
- data/spec/shoryuken_spec.rb +9 -0
- metadata +9 -7
- data/Gemfile.aws-sdk-core-v2 +0 -13
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1002845b2232c5f7491b2839d7851be5afdb79a3a11e73e2249b81e1f2ea4ec4
|
4
|
+
data.tar.gz: 8398fa557fba47360763bb83ea4e0f529b2c69bad8b15fb3163e03c517b460d8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 010d2f45fe45e6967435b57c1f0c7601f9d3a43a8c1108edbf97b303e60e108e0c6be1ba28915a43bc84f5904bd7f3d634916251b8b1c0d83143d75bdccaaca7
|
7
|
+
data.tar.gz: 00c8f19de3a9fd241bb236023479bd5c3bd27f731001f036377c05b75c18bce119b08666eb13649606904e2950f937f52615eba84facb44074099fde1fc97534
|
data/.github/workflows/specs.yml
CHANGED
@@ -9,8 +9,8 @@ jobs:
|
|
9
9
|
name: All Specs
|
10
10
|
strategy:
|
11
11
|
matrix:
|
12
|
-
ruby: ['2.4
|
13
|
-
gemfile: ['Gemfile', '
|
12
|
+
ruby: ['2.4', '2.5', '2.6', '2.7', '3.0']
|
13
|
+
gemfile: ['Gemfile', 'gemfiles/aws_sdk_core_2.gemfile']
|
14
14
|
runs-on: ubuntu-20.04
|
15
15
|
services:
|
16
16
|
moto_sqs:
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
data/Appraisals
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,29 @@
|
|
1
|
+
## [v5.3.2] - 2022-01-19
|
2
|
+
|
3
|
+
- (Bugfix) Preserve queue weights when unpausing queues
|
4
|
+
- [#687](https://github.com/ruby-shoryuken/shoryuken/pull/687)
|
5
|
+
|
6
|
+
- Improve error message on startup when shoryuken has insufficient permissions to access a queue
|
7
|
+
- [#691](https://github.com/ruby-shoryuken/shoryuken/pull/691)
|
8
|
+
|
9
|
+
## [v5.3.1] - 2022-01-07
|
10
|
+
|
11
|
+
- (Bugfix) Fix issue where, when using the TSTP or USR1 signals for soft shutdowns, it was possible for shoryuken to terminate without first attempting to handle all messages it fetched from SQS
|
12
|
+
- [#676](https://github.com/ruby-shoryuken/shoryuken/pull/676)
|
13
|
+
|
14
|
+
## [v5.3.0] - 2021-10-31
|
15
|
+
|
16
|
+
- (Refactor) Use Forwardable within Message to avoid method boilerplate
|
17
|
+
- [#681](https://github.com/ruby-shoryuken/shoryuken/pull/681)
|
18
|
+
|
19
|
+
- Add basic health check API
|
20
|
+
- [#679](https://github.com/ruby-shoryuken/shoryuken/pull/679)
|
21
|
+
|
22
|
+
## [v5.2.3] - 2021-07-29
|
23
|
+
|
24
|
+
- Fire new `:utilization_update` event any time a worker pool's utilization changes
|
25
|
+
- [#673](https://github.com/ruby-shoryuken/shoryuken/pull/673)
|
26
|
+
|
1
27
|
## [v5.2.2] - 2021-06-22
|
2
28
|
|
3
29
|
- When using ActiveJob queue name prefixing, avoid applying prefix to queues configured with a URL or ARN
|
data/Gemfile
CHANGED
@@ -0,0 +1,21 @@
|
|
1
|
+
# This file was generated by Appraisal
|
2
|
+
|
3
|
+
source "https://rubygems.org"
|
4
|
+
|
5
|
+
group :test do
|
6
|
+
gem "activejob"
|
7
|
+
gem "aws-sdk-core", "~> 2"
|
8
|
+
gem "codeclimate-test-reporter", require: nil
|
9
|
+
gem "httparty"
|
10
|
+
gem "multi_xml"
|
11
|
+
gem "simplecov"
|
12
|
+
gem "webrick"
|
13
|
+
end
|
14
|
+
|
15
|
+
group :development do
|
16
|
+
gem "appraisal", git: "https://github.com/thoughtbot/appraisal.git"
|
17
|
+
gem "pry-byebug", "3.9.0"
|
18
|
+
gem "rubocop"
|
19
|
+
end
|
20
|
+
|
21
|
+
gemspec path: "../"
|
@@ -159,9 +159,17 @@ module Shoryuken
|
|
159
159
|
|
160
160
|
return if non_existent_queues.none?
|
161
161
|
|
162
|
+
# NOTE: HEREDOC's ~ operator removes indents, but is only available Ruby 2.3+
|
163
|
+
# See github PR: https://github.com/ruby-shoryuken/shoryuken/pull/691#issuecomment-1007653595
|
164
|
+
error_msg = <<-MSG.gsub(/^\s+/, '')
|
165
|
+
The specified queue(s) #{non_existent_queues.join(', ')} do not exist.
|
166
|
+
Try 'shoryuken sqs create QUEUE-NAME' for creating a queue with default settings.
|
167
|
+
It's also possible that you don't have permission to access the specified queues.
|
168
|
+
MSG
|
169
|
+
|
162
170
|
fail(
|
163
171
|
ArgumentError,
|
164
|
-
|
172
|
+
error_msg
|
165
173
|
)
|
166
174
|
end
|
167
175
|
|
data/lib/shoryuken/launcher.rb
CHANGED
@@ -28,12 +28,30 @@ module Shoryuken
|
|
28
28
|
|
29
29
|
initiate_stop
|
30
30
|
|
31
|
+
stop_new_dispatching
|
32
|
+
await_dispatching_in_progress
|
33
|
+
|
31
34
|
executor.shutdown
|
32
35
|
executor.wait_for_termination
|
33
36
|
end
|
34
37
|
|
38
|
+
def healthy?
|
39
|
+
Shoryuken.groups.keys.all? do |group|
|
40
|
+
manager = @managers.find { |m| m.group == group }
|
41
|
+
manager && manager.running?
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
35
45
|
private
|
36
46
|
|
47
|
+
def stop_new_dispatching
|
48
|
+
@managers.each(&:stop_new_dispatching)
|
49
|
+
end
|
50
|
+
|
51
|
+
def await_dispatching_in_progress
|
52
|
+
@managers.each(&:await_dispatching_in_progress)
|
53
|
+
end
|
54
|
+
|
37
55
|
def executor
|
38
56
|
@_executor ||= Shoryuken.launcher_executor || Concurrent.global_io_executor
|
39
57
|
end
|
@@ -71,6 +89,7 @@ module Shoryuken
|
|
71
89
|
def create_managers
|
72
90
|
Shoryuken.groups.map do |group, options|
|
73
91
|
Shoryuken::Manager.new(
|
92
|
+
group,
|
74
93
|
Shoryuken::Fetcher.new(group),
|
75
94
|
Shoryuken.polling_strategy(group).new(options[:queues], Shoryuken.delay(group)),
|
76
95
|
options[:concurrency],
|
data/lib/shoryuken/manager.rb
CHANGED
@@ -6,27 +6,47 @@ module Shoryuken
|
|
6
6
|
# See https://github.com/phstc/shoryuken/issues/348#issuecomment-292847028
|
7
7
|
MIN_DISPATCH_INTERVAL = 0.1
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
@
|
13
|
-
@
|
14
|
-
@
|
15
|
-
@
|
9
|
+
attr_reader :group
|
10
|
+
|
11
|
+
def initialize(group, fetcher, polling_strategy, concurrency, executor)
|
12
|
+
@group = group
|
13
|
+
@fetcher = fetcher
|
14
|
+
@polling_strategy = polling_strategy
|
15
|
+
@max_processors = concurrency
|
16
|
+
@busy_processors = Concurrent::AtomicFixnum.new(0)
|
17
|
+
@executor = executor
|
18
|
+
@running = Concurrent::AtomicBoolean.new(true)
|
19
|
+
@stop_new_dispatching = Concurrent::AtomicBoolean.new(false)
|
20
|
+
@dispatching_release_signal = ::Queue.new
|
16
21
|
end
|
17
22
|
|
18
23
|
def start
|
24
|
+
fire_utilization_update_event
|
19
25
|
dispatch_loop
|
20
26
|
end
|
21
27
|
|
22
|
-
|
28
|
+
def stop_new_dispatching
|
29
|
+
@stop_new_dispatching.make_true
|
30
|
+
end
|
31
|
+
|
32
|
+
def await_dispatching_in_progress
|
33
|
+
# There might still be a dispatching on-going, as the response from SQS could take some time
|
34
|
+
# We don't want to stop the process before processing incoming messages, as they would stay "in-flight" for some time on SQS
|
35
|
+
# We use a queue, as the dispatch_loop is running on another thread, and this is a efficient way of communicating between threads.
|
36
|
+
@dispatching_release_signal.pop
|
37
|
+
end
|
23
38
|
|
24
39
|
def running?
|
25
40
|
@running.true? && @executor.running?
|
26
41
|
end
|
27
42
|
|
43
|
+
private
|
44
|
+
|
28
45
|
def dispatch_loop
|
29
|
-
|
46
|
+
if @stop_new_dispatching.true? || !running?
|
47
|
+
@dispatching_release_signal << 1
|
48
|
+
return
|
49
|
+
end
|
30
50
|
|
31
51
|
@executor.post { dispatch }
|
32
52
|
end
|
@@ -59,6 +79,8 @@ module Shoryuken
|
|
59
79
|
|
60
80
|
def processor_done(queue)
|
61
81
|
@busy_processors.decrement
|
82
|
+
fire_utilization_update_event
|
83
|
+
|
62
84
|
client_queue = Shoryuken::Client.queues(queue)
|
63
85
|
return unless client_queue.fifo?
|
64
86
|
return unless @polling_strategy.respond_to?(:message_processed)
|
@@ -72,6 +94,7 @@ module Shoryuken
|
|
72
94
|
logger.debug { "Assigning #{sqs_msg.message_id}" }
|
73
95
|
|
74
96
|
@busy_processors.increment
|
97
|
+
fire_utilization_update_event
|
75
98
|
|
76
99
|
Concurrent::Promise
|
77
100
|
.execute(executor: @executor) { Processor.process(queue_name, sqs_msg) }
|
@@ -87,7 +110,6 @@ module Shoryuken
|
|
87
110
|
|
88
111
|
def dispatch_single_messages(queue)
|
89
112
|
messages = @fetcher.fetch(queue, ready)
|
90
|
-
|
91
113
|
@polling_strategy.messages_found(queue.name, messages.size)
|
92
114
|
messages.each { |message| assign(queue.name, message) }
|
93
115
|
end
|
@@ -114,5 +136,13 @@ module Shoryuken
|
|
114
136
|
|
115
137
|
@running.make_false
|
116
138
|
end
|
139
|
+
|
140
|
+
def fire_utilization_update_event
|
141
|
+
fire_event :utilization_update, false, {
|
142
|
+
group: @group,
|
143
|
+
max_processors: @max_processors,
|
144
|
+
busy_processors: busy
|
145
|
+
}
|
146
|
+
end
|
117
147
|
end
|
118
148
|
end
|
data/lib/shoryuken/message.rb
CHANGED
@@ -1,5 +1,16 @@
|
|
1
1
|
module Shoryuken
|
2
2
|
class Message
|
3
|
+
extend Forwardable
|
4
|
+
|
5
|
+
def_delegators(:data,
|
6
|
+
:message_id,
|
7
|
+
:receipt_handle,
|
8
|
+
:md5_of_body,
|
9
|
+
:body,
|
10
|
+
:attributes,
|
11
|
+
:md5_of_message_attributes,
|
12
|
+
:message_attributes)
|
13
|
+
|
3
14
|
attr_accessor :client, :queue_url, :queue_name, :data
|
4
15
|
|
5
16
|
def initialize(client, queue, data)
|
@@ -29,33 +40,5 @@ module Shoryuken
|
|
29
40
|
visibility_timeout: timeout
|
30
41
|
)
|
31
42
|
end
|
32
|
-
|
33
|
-
def message_id
|
34
|
-
data.message_id
|
35
|
-
end
|
36
|
-
|
37
|
-
def receipt_handle
|
38
|
-
data.receipt_handle
|
39
|
-
end
|
40
|
-
|
41
|
-
def md5_of_body
|
42
|
-
data.md5_of_body
|
43
|
-
end
|
44
|
-
|
45
|
-
def body
|
46
|
-
data.body
|
47
|
-
end
|
48
|
-
|
49
|
-
def attributes
|
50
|
-
data.attributes
|
51
|
-
end
|
52
|
-
|
53
|
-
def md5_of_message_attributes
|
54
|
-
data.md5_of_message_attributes
|
55
|
-
end
|
56
|
-
|
57
|
-
def message_attributes
|
58
|
-
data.message_attributes
|
59
|
-
end
|
60
43
|
end
|
61
44
|
end
|
data/lib/shoryuken/options.rb
CHANGED
@@ -39,8 +39,10 @@ module Shoryuken
|
|
39
39
|
end
|
40
40
|
|
41
41
|
def message_processed(queue)
|
42
|
-
|
43
|
-
|
42
|
+
if queue_paused?(queue)
|
43
|
+
logger.debug "Unpausing #{queue}"
|
44
|
+
@paused_until[queue] = Time.at 0
|
45
|
+
end
|
44
46
|
end
|
45
47
|
|
46
48
|
private
|
@@ -36,12 +36,10 @@ module Shoryuken
|
|
36
36
|
end
|
37
37
|
|
38
38
|
def message_processed(queue)
|
39
|
-
|
39
|
+
paused_queue = @paused_queues.find { |_time, name| name == queue }
|
40
|
+
return unless paused_queue
|
40
41
|
|
41
|
-
|
42
|
-
@paused_queues.reject! { |_time, name| name == queue }
|
43
|
-
@queues << queue
|
44
|
-
@queues.uniq!
|
42
|
+
paused_queue[0] = Time.at 0
|
45
43
|
end
|
46
44
|
|
47
45
|
private
|
data/lib/shoryuken/runner.rb
CHANGED
data/lib/shoryuken/version.rb
CHANGED
data/lib/shoryuken.rb
CHANGED
@@ -4,6 +4,36 @@ require 'active_job'
|
|
4
4
|
RSpec.describe Shoryuken::EnvironmentLoader do
|
5
5
|
subject { described_class.new({}) }
|
6
6
|
|
7
|
+
describe '#load' do
|
8
|
+
before do
|
9
|
+
Shoryuken.groups.clear
|
10
|
+
# See issue: https://stackoverflow.com/a/63699568 for stubbing AWS errors
|
11
|
+
allow(Shoryuken::Client)
|
12
|
+
.to receive(:queues)
|
13
|
+
.with('stubbed_queue')
|
14
|
+
.and_raise(Aws::SQS::Errors::NonExistentQueue.new(nil, nil))
|
15
|
+
allow(subject).to receive(:load_rails)
|
16
|
+
allow(subject).to receive(:prefix_active_job_queue_names)
|
17
|
+
allow(subject).to receive(:require_workers)
|
18
|
+
allow(subject).to receive(:validate_workers)
|
19
|
+
allow(subject).to receive(:patch_deprecated_workers)
|
20
|
+
Shoryuken.options[:groups] = [['custom', { queues: ['stubbed_queue'] }]]
|
21
|
+
end
|
22
|
+
|
23
|
+
context "when given queues don't exist" do
|
24
|
+
specify do
|
25
|
+
expect { subject.load }.to raise_error(
|
26
|
+
ArgumentError,
|
27
|
+
<<-MSG.gsub(/^\s+/, '')
|
28
|
+
The specified queue(s) stubbed_queue do not exist.
|
29
|
+
Try 'shoryuken sqs create QUEUE-NAME' for creating a queue with default settings.
|
30
|
+
It's also possible that you don't have permission to access the specified queues.
|
31
|
+
MSG
|
32
|
+
)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
7
37
|
describe '#parse_queues loads default queues' do
|
8
38
|
before do
|
9
39
|
allow(subject).to receive(:load_rails)
|
@@ -34,7 +64,7 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
34
64
|
|
35
65
|
specify do
|
36
66
|
Shoryuken.options[:queues] = ['queue1', 'queue2'] # default queues
|
37
|
-
Shoryuken.options[:groups] = [[
|
67
|
+
Shoryuken.options[:groups] = [['custom', { queues: ['queue3'], delay: 25 }]]
|
38
68
|
subject.load
|
39
69
|
|
40
70
|
expect(Shoryuken.groups['default'][:queues]).to eq(%w[queue1 queue2])
|
@@ -44,7 +74,6 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
44
74
|
end
|
45
75
|
end
|
46
76
|
|
47
|
-
|
48
77
|
describe '#prefix_active_job_queue_names' do
|
49
78
|
before do
|
50
79
|
allow(subject).to receive(:load_rails)
|
@@ -76,7 +105,7 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
76
105
|
|
77
106
|
it 'does not prefix url-based queues' do
|
78
107
|
Shoryuken.options[:queues] = ['https://example.com/test_queue1']
|
79
|
-
Shoryuken.options[:groups] = {'group1' => {queues: ['https://example.com/test_group1_queue1']}}
|
108
|
+
Shoryuken.options[:groups] = { 'group1' => { queues: ['https://example.com/test_group1_queue1'] } }
|
80
109
|
|
81
110
|
subject.load
|
82
111
|
|
@@ -86,7 +115,7 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
86
115
|
|
87
116
|
it 'does not prefix arn-based queues' do
|
88
117
|
Shoryuken.options[:queues] = ['arn:aws:sqs:fake-region-1:1234:test_queue1']
|
89
|
-
Shoryuken.options[:groups] = {'group1' => {queues: ['arn:aws:sqs:fake-region-1:1234:test_group1_queue1']}}
|
118
|
+
Shoryuken.options[:groups] = { 'group1' => { queues: ['arn:aws:sqs:fake-region-1:1234:test_group1_queue1'] } }
|
90
119
|
|
91
120
|
subject.load
|
92
121
|
|
@@ -94,9 +123,11 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
94
123
|
expect(Shoryuken.groups['group1'][:queues]).to(eq(['arn:aws:sqs:fake-region-1:1234:test_group1_queue1']))
|
95
124
|
end
|
96
125
|
end
|
126
|
+
|
97
127
|
describe "#setup_options" do
|
98
|
-
let
|
99
|
-
let
|
128
|
+
let(:cli_queues) { { "queue1" => 10, "queue2" => 20 } }
|
129
|
+
let(:config_queues) { [["queue1", 8], ["queue2", 4]] }
|
130
|
+
|
100
131
|
context "when given queues through config and CLI" do
|
101
132
|
specify do
|
102
133
|
allow_any_instance_of(Shoryuken::EnvironmentLoader).to receive(:config_file_options).and_return({ queues: config_queues })
|
@@ -104,6 +135,7 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
104
135
|
expect(Shoryuken.options[:queues]).to eq(cli_queues)
|
105
136
|
end
|
106
137
|
end
|
138
|
+
|
107
139
|
context "when given queues through config only" do
|
108
140
|
specify do
|
109
141
|
allow_any_instance_of(Shoryuken::EnvironmentLoader).to receive(:config_file_options).and_return({ queues: config_queues })
|
@@ -111,6 +143,7 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
111
143
|
expect(Shoryuken.options[:queues]).to eq(config_queues)
|
112
144
|
end
|
113
145
|
end
|
146
|
+
|
114
147
|
context "when given queues through CLI only" do
|
115
148
|
specify do
|
116
149
|
Shoryuken::EnvironmentLoader.setup_options(queues: cli_queues)
|
@@ -0,0 +1,60 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'shoryuken/launcher'
|
3
|
+
|
4
|
+
RSpec.describe Shoryuken::Launcher do
|
5
|
+
let(:executor) do
|
6
|
+
# We can't use Concurrent.global_io_executor in these tests since once you
|
7
|
+
# shut down a thread pool, you can't start it back up. Instead, we create
|
8
|
+
# one new thread pool executor for each spec. We use a new
|
9
|
+
# CachedThreadPool, since that most closely resembles
|
10
|
+
# Concurrent.global_io_executor
|
11
|
+
Concurrent::CachedThreadPool.new auto_terminate: true
|
12
|
+
end
|
13
|
+
|
14
|
+
let(:first_group_manager) { double(:first_group_manager, group: 'first_group') }
|
15
|
+
let(:second_group_manager) { double(:second_group_manager, group: 'second_group') }
|
16
|
+
let(:first_queue) { "launcher_spec_#{SecureRandom.uuid}" }
|
17
|
+
let(:second_queue) { "launcher_spec_#{SecureRandom.uuid}" }
|
18
|
+
|
19
|
+
before do
|
20
|
+
Shoryuken.add_group('first_group', 1)
|
21
|
+
Shoryuken.add_group('second_group', 1)
|
22
|
+
Shoryuken.add_queue(first_queue, 1, 'first_group')
|
23
|
+
Shoryuken.add_queue(second_queue, 1, 'second_group')
|
24
|
+
allow(Shoryuken).to receive(:launcher_executor).and_return(executor)
|
25
|
+
allow(Shoryuken::Manager).to receive(:new).with('first_group', any_args).and_return(first_group_manager)
|
26
|
+
allow(Shoryuken::Manager).to receive(:new).with('second_group', any_args).and_return(second_group_manager)
|
27
|
+
allow(first_group_manager).to receive(:running?).and_return(true)
|
28
|
+
allow(second_group_manager).to receive(:running?).and_return(true)
|
29
|
+
end
|
30
|
+
|
31
|
+
describe '#healthy?' do
|
32
|
+
context 'when all groups have managers' do
|
33
|
+
context 'when all managers are running' do
|
34
|
+
it 'returns true' do
|
35
|
+
expect(subject.healthy?).to be true
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
context 'when one manager is not running' do
|
40
|
+
before do
|
41
|
+
allow(second_group_manager).to receive(:running?).and_return(false)
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'returns false' do
|
45
|
+
expect(subject.healthy?).to be false
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
context 'when all groups do not have managers' do
|
51
|
+
before do
|
52
|
+
allow(second_group_manager).to receive(:group).and_return('some_random_group')
|
53
|
+
end
|
54
|
+
|
55
|
+
it 'returns false' do
|
56
|
+
expect(subject.healthy?).to be false
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
@@ -15,7 +15,7 @@ RSpec.describe Shoryuken::Manager do
|
|
15
15
|
let(:concurrency) { 1 }
|
16
16
|
let(:executor) { Concurrent::ImmediateExecutor.new }
|
17
17
|
|
18
|
-
subject { Shoryuken::Manager.new(fetcher, polling_strategy, concurrency, executor) }
|
18
|
+
subject { Shoryuken::Manager.new('default', fetcher, polling_strategy, concurrency, executor) }
|
19
19
|
|
20
20
|
before do
|
21
21
|
allow(fetcher).to receive(:fetch).and_return([])
|
@@ -71,6 +71,11 @@ RSpec.describe Shoryuken::Manager do
|
|
71
71
|
|
72
72
|
expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
|
73
73
|
expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
|
74
|
+
expect(subject).to receive(:fire_event).with(:utilization_update,
|
75
|
+
false,
|
76
|
+
group: 'default',
|
77
|
+
busy_processors: 1,
|
78
|
+
max_processors: 1)
|
74
79
|
expect(Shoryuken::Processor).to receive(:process).with(q, message)
|
75
80
|
expect(Shoryuken.logger).to receive(:info).never
|
76
81
|
|
@@ -99,6 +104,11 @@ RSpec.describe Shoryuken::Manager do
|
|
99
104
|
q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
|
100
105
|
|
101
106
|
expect(fetcher).to receive(:fetch).with(q, described_class::BATCH_LIMIT).and_return(messages)
|
107
|
+
expect(subject).to receive(:fire_event).with(:utilization_update,
|
108
|
+
false,
|
109
|
+
group: 'default',
|
110
|
+
busy_processors: 1,
|
111
|
+
max_processors: 1)
|
102
112
|
expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
|
103
113
|
allow(subject).to receive(:batched_queue?).with(q).and_return(true)
|
104
114
|
expect(Shoryuken::Processor).to receive(:process).with(q, messages)
|
@@ -163,4 +173,26 @@ RSpec.describe Shoryuken::Manager do
|
|
163
173
|
end
|
164
174
|
end
|
165
175
|
end
|
176
|
+
|
177
|
+
describe '#running?' do
|
178
|
+
context 'when the executor is running' do
|
179
|
+
before do
|
180
|
+
allow(executor).to receive(:running?).and_return(true)
|
181
|
+
end
|
182
|
+
|
183
|
+
it 'returns true' do
|
184
|
+
expect(subject.running?).to be true
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
context 'when the executor is not running' do
|
189
|
+
before do
|
190
|
+
allow(executor).to receive(:running?).and_return(false)
|
191
|
+
end
|
192
|
+
|
193
|
+
it 'returns false' do
|
194
|
+
expect(subject.running?).to be false
|
195
|
+
end
|
196
|
+
end
|
197
|
+
end
|
166
198
|
end
|
@@ -106,12 +106,37 @@ RSpec.describe Shoryuken::Polling::WeightedRoundRobin do
|
|
106
106
|
end
|
107
107
|
|
108
108
|
describe '#message_processed' do
|
109
|
-
it 'removes paused queue
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
109
|
+
it 'removes delay from paused queue' do
|
110
|
+
queues << queue1
|
111
|
+
queues << queue2
|
112
|
+
|
113
|
+
expect(subject.next_queue).to eq(queue1)
|
114
|
+
subject.messages_found(queue1, 0) # pauses queue1
|
115
|
+
|
116
|
+
expect(subject.active_queues).to eq([[queue2, 1]])
|
117
|
+
|
118
|
+
subject.message_processed(queue1) # marks queue1 to be unpaused
|
119
|
+
|
120
|
+
expect(subject.next_queue).to eq(queue2) # implicitly unpauses queue1
|
121
|
+
expect(subject.active_queues).to eq([[queue1, 1], [queue2, 1]])
|
122
|
+
end
|
123
|
+
|
124
|
+
it 'preserves weight of queues when unpausing' do
|
125
|
+
queues << queue1
|
126
|
+
queues << queue1
|
127
|
+
queues << queue2
|
128
|
+
|
129
|
+
expect(subject.next_queue).to eq(queue1)
|
130
|
+
subject.messages_found(queue1, 1)
|
131
|
+
|
132
|
+
expect(subject.next_queue).to eq(queue2)
|
133
|
+
subject.messages_found(queue2, 0) # pauses queue2
|
134
|
+
|
135
|
+
expect(subject.active_queues).to eq([[queue1, 2]])
|
136
|
+
subject.message_processed(queue2) # marks queue2 to be unpaused
|
137
|
+
|
138
|
+
expect(subject.next_queue).to eq(queue1) # implicitly unpauses queue2
|
139
|
+
expect(subject.active_queues).to eq([[queue1, 2], [queue2, 1]])
|
115
140
|
end
|
116
141
|
end
|
117
142
|
end
|
data/spec/shoryuken_spec.rb
CHANGED
@@ -1,4 +1,13 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
RSpec.describe Shoryuken do
|
4
|
+
describe '.healthy?' do
|
5
|
+
before do
|
6
|
+
allow(Shoryuken::Runner).to receive(:instance).and_return(double(:instance, healthy?: :some_result))
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'delegates to the runner instance' do
|
10
|
+
expect(described_class.healthy?).to eq(:some_result)
|
11
|
+
end
|
12
|
+
end
|
4
13
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shoryuken
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.3.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pablo Cantero
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-01-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: dotenv
|
@@ -112,7 +112,6 @@ files:
|
|
112
112
|
- Appraisals
|
113
113
|
- CHANGELOG.md
|
114
114
|
- Gemfile
|
115
|
-
- Gemfile.aws-sdk-core-v2
|
116
115
|
- LICENSE
|
117
116
|
- README.md
|
118
117
|
- Rakefile
|
@@ -122,6 +121,7 @@ files:
|
|
122
121
|
- examples/bootstrap_queues.rb
|
123
122
|
- examples/default_worker.rb
|
124
123
|
- gemfiles/.gitignore
|
124
|
+
- gemfiles/aws_sdk_core_2.gemfile
|
125
125
|
- gemfiles/rails_4_2.gemfile
|
126
126
|
- gemfiles/rails_5_2.gemfile
|
127
127
|
- gemfiles/rails_6_0.gemfile
|
@@ -174,6 +174,7 @@ files:
|
|
174
174
|
- spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb
|
175
175
|
- spec/shoryuken/extensions/active_job_wrapper_spec.rb
|
176
176
|
- spec/shoryuken/fetcher_spec.rb
|
177
|
+
- spec/shoryuken/launcher_spec.rb
|
177
178
|
- spec/shoryuken/manager_spec.rb
|
178
179
|
- spec/shoryuken/middleware/chain_spec.rb
|
179
180
|
- spec/shoryuken/middleware/server/auto_delete_spec.rb
|
@@ -198,7 +199,7 @@ homepage: https://github.com/phstc/shoryuken
|
|
198
199
|
licenses:
|
199
200
|
- LGPL-3.0
|
200
201
|
metadata: {}
|
201
|
-
post_install_message:
|
202
|
+
post_install_message:
|
202
203
|
rdoc_options: []
|
203
204
|
require_paths:
|
204
205
|
- lib
|
@@ -213,8 +214,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
213
214
|
- !ruby/object:Gem::Version
|
214
215
|
version: '0'
|
215
216
|
requirements: []
|
216
|
-
rubygems_version: 3.0.1
|
217
|
-
signing_key:
|
217
|
+
rubygems_version: 3.0.3.1
|
218
|
+
signing_key:
|
218
219
|
specification_version: 4
|
219
220
|
summary: Shoryuken is a super efficient AWS SQS thread based message processor
|
220
221
|
test_files:
|
@@ -231,6 +232,7 @@ test_files:
|
|
231
232
|
- spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb
|
232
233
|
- spec/shoryuken/extensions/active_job_wrapper_spec.rb
|
233
234
|
- spec/shoryuken/fetcher_spec.rb
|
235
|
+
- spec/shoryuken/launcher_spec.rb
|
234
236
|
- spec/shoryuken/manager_spec.rb
|
235
237
|
- spec/shoryuken/middleware/chain_spec.rb
|
236
238
|
- spec/shoryuken/middleware/server/auto_delete_spec.rb
|
data/Gemfile.aws-sdk-core-v2
DELETED
@@ -1,13 +0,0 @@
|
|
1
|
-
source 'https://rubygems.org'
|
2
|
-
|
3
|
-
# Specify your gem's dependencies in shoryuken.gemspec
|
4
|
-
gemspec
|
5
|
-
|
6
|
-
group :test do
|
7
|
-
gem 'activejob', '~> 4'
|
8
|
-
gem 'aws-sdk-core', '~> 2'
|
9
|
-
gem 'codeclimate-test-reporter', require: nil
|
10
|
-
gem 'httparty'
|
11
|
-
gem 'multi_xml'
|
12
|
-
gem 'simplecov'
|
13
|
-
end
|