shoryuken 5.1.1 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.devcontainer/Dockerfile +17 -0
- data/.devcontainer/base.Dockerfile +43 -0
- data/.devcontainer/devcontainer.json +35 -0
- data/.github/workflows/specs.yml +10 -5
- data/.github/workflows/stale.yml +20 -0
- data/.gitignore +1 -1
- data/.rubocop.yml +1 -1
- data/Appraisals +8 -0
- data/CHANGELOG.md +57 -0
- data/Gemfile +1 -1
- data/README.md +8 -2
- data/Rakefile +14 -3
- data/bin/cli/sqs.rb +50 -5
- data/gemfiles/aws_sdk_core_2.gemfile +21 -0
- data/lib/shoryuken/environment_loader.rb +17 -3
- data/lib/shoryuken/extensions/active_job_adapter.rb +4 -2
- data/lib/shoryuken/launcher.rb +19 -0
- data/lib/shoryuken/manager.rb +40 -10
- data/lib/shoryuken/message.rb +11 -28
- data/lib/shoryuken/options.rb +1 -0
- data/lib/shoryuken/polling/strict_priority.rb +4 -2
- data/lib/shoryuken/polling/weighted_round_robin.rb +3 -5
- data/lib/shoryuken/runner.rb +4 -3
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken.rb +4 -0
- data/spec/integration/launcher_spec.rb +29 -2
- data/spec/shoryuken/environment_loader_spec.rb +62 -9
- data/spec/shoryuken/extensions/active_job_concurrent_send_adapter_spec.rb +4 -0
- data/spec/shoryuken/extensions/active_job_wrapper_spec.rb +20 -0
- data/spec/shoryuken/launcher_spec.rb +60 -0
- data/spec/shoryuken/manager_spec.rb +37 -1
- data/spec/shoryuken/polling/weighted_round_robin_spec.rb +31 -6
- data/spec/shoryuken/queue_spec.rb +10 -5
- data/spec/shoryuken/worker/default_executor_spec.rb +48 -48
- data/spec/shoryuken_spec.rb +9 -0
- data/spec/spec_helper.rb +0 -4
- metadata +12 -4
- data/Gemfile.aws-sdk-core-v2 +0 -13
data/lib/shoryuken/options.rb
CHANGED
|
@@ -39,8 +39,10 @@ module Shoryuken
|
|
|
39
39
|
end
|
|
40
40
|
|
|
41
41
|
def message_processed(queue)
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
if queue_paused?(queue)
|
|
43
|
+
logger.debug "Unpausing #{queue}"
|
|
44
|
+
@paused_until[queue] = Time.at 0
|
|
45
|
+
end
|
|
44
46
|
end
|
|
45
47
|
|
|
46
48
|
private
|
|
@@ -36,12 +36,10 @@ module Shoryuken
|
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
def message_processed(queue)
|
|
39
|
-
|
|
39
|
+
paused_queue = @paused_queues.find { |_time, name| name == queue }
|
|
40
|
+
return unless paused_queue
|
|
40
41
|
|
|
41
|
-
|
|
42
|
-
@paused_queues.reject! { |_time, name| name == queue }
|
|
43
|
-
@queues << queue
|
|
44
|
-
@queues.uniq!
|
|
42
|
+
paused_queue[0] = Time.at 0
|
|
45
43
|
end
|
|
46
44
|
|
|
47
45
|
private
|
data/lib/shoryuken/runner.rb
CHANGED
|
@@ -30,9 +30,6 @@ module Shoryuken
|
|
|
30
30
|
|
|
31
31
|
loader = EnvironmentLoader.setup_options(options)
|
|
32
32
|
|
|
33
|
-
# When cli args exist, override options in config file
|
|
34
|
-
Shoryuken.options.merge!(options)
|
|
35
|
-
|
|
36
33
|
daemonize(Shoryuken.options)
|
|
37
34
|
write_pid(Shoryuken.options)
|
|
38
35
|
|
|
@@ -55,6 +52,10 @@ module Shoryuken
|
|
|
55
52
|
end
|
|
56
53
|
end
|
|
57
54
|
|
|
55
|
+
def healthy?
|
|
56
|
+
(@launcher && @launcher.healthy?) || false
|
|
57
|
+
end
|
|
58
|
+
|
|
58
59
|
private
|
|
59
60
|
|
|
60
61
|
def initialize_concurrent_logger
|
data/lib/shoryuken/version.rb
CHANGED
data/lib/shoryuken.rb
CHANGED
|
@@ -4,10 +4,37 @@ require 'shoryuken/launcher'
|
|
|
4
4
|
require 'securerandom'
|
|
5
5
|
|
|
6
6
|
RSpec.describe Shoryuken::Launcher do
|
|
7
|
-
|
|
7
|
+
let(:sqs_client) do
|
|
8
|
+
Aws::SQS::Client.new(
|
|
9
|
+
region: 'us-east-1',
|
|
10
|
+
endpoint: 'http://localhost:5000',
|
|
11
|
+
access_key_id: 'fake',
|
|
12
|
+
secret_access_key: 'fake'
|
|
13
|
+
)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
let(:executor) do
|
|
17
|
+
# We can't use Concurrent.global_io_executor in these tests since once you
|
|
18
|
+
# shut down a thread pool, you can't start it back up. Instead, we create
|
|
19
|
+
# one new thread pool executor for each spec. We use a new
|
|
20
|
+
# CachedThreadPool, since that most closely resembles
|
|
21
|
+
# Concurrent.global_io_executor
|
|
22
|
+
Concurrent::CachedThreadPool.new auto_terminate: true
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
describe 'Consuming messages' do
|
|
8
26
|
before do
|
|
9
27
|
Aws.config[:stub_responses] = false
|
|
10
|
-
|
|
28
|
+
|
|
29
|
+
allow(Shoryuken).to receive(:launcher_executor).and_return(executor)
|
|
30
|
+
|
|
31
|
+
Shoryuken.configure_client do |config|
|
|
32
|
+
config.sqs_client = sqs_client
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
Shoryuken.configure_server do |config|
|
|
36
|
+
config.sqs_client = sqs_client
|
|
37
|
+
end
|
|
11
38
|
|
|
12
39
|
StandardWorker.received_messages = 0
|
|
13
40
|
|
|
@@ -4,11 +4,41 @@ require 'active_job'
|
|
|
4
4
|
RSpec.describe Shoryuken::EnvironmentLoader do
|
|
5
5
|
subject { described_class.new({}) }
|
|
6
6
|
|
|
7
|
-
describe '#
|
|
7
|
+
describe '#load' do
|
|
8
8
|
before do
|
|
9
|
+
Shoryuken.groups.clear
|
|
10
|
+
# See issue: https://stackoverflow.com/a/63699568 for stubbing AWS errors
|
|
11
|
+
allow(Shoryuken::Client)
|
|
12
|
+
.to receive(:queues)
|
|
13
|
+
.with('stubbed_queue')
|
|
14
|
+
.and_raise(Aws::SQS::Errors::NonExistentQueue.new(nil, nil))
|
|
9
15
|
allow(subject).to receive(:load_rails)
|
|
10
16
|
allow(subject).to receive(:prefix_active_job_queue_names)
|
|
11
17
|
allow(subject).to receive(:require_workers)
|
|
18
|
+
allow(subject).to receive(:validate_workers)
|
|
19
|
+
allow(subject).to receive(:patch_deprecated_workers)
|
|
20
|
+
Shoryuken.options[:groups] = [['custom', { queues: ['stubbed_queue'] }]]
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
context "when given queues don't exist" do
|
|
24
|
+
specify do
|
|
25
|
+
expect { subject.load }.to raise_error(
|
|
26
|
+
ArgumentError,
|
|
27
|
+
<<-MSG.gsub(/^\s+/, '')
|
|
28
|
+
The specified queue(s) stubbed_queue do not exist.
|
|
29
|
+
Try 'shoryuken sqs create QUEUE-NAME' for creating a queue with default settings.
|
|
30
|
+
It's also possible that you don't have permission to access the specified queues.
|
|
31
|
+
MSG
|
|
32
|
+
)
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
describe '#parse_queues loads default queues' do
|
|
38
|
+
before do
|
|
39
|
+
allow(subject).to receive(:initialize_rails)
|
|
40
|
+
allow(subject).to receive(:prefix_active_job_queue_names)
|
|
41
|
+
allow(subject).to receive(:require_workers)
|
|
12
42
|
allow(subject).to receive(:validate_queues)
|
|
13
43
|
allow(subject).to receive(:validate_workers)
|
|
14
44
|
allow(subject).to receive(:patch_deprecated_workers)
|
|
@@ -24,7 +54,7 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
|
24
54
|
|
|
25
55
|
describe '#parse_queues includes delay per groups' do
|
|
26
56
|
before do
|
|
27
|
-
allow(subject).to receive(:
|
|
57
|
+
allow(subject).to receive(:initialize_rails)
|
|
28
58
|
allow(subject).to receive(:prefix_active_job_queue_names)
|
|
29
59
|
allow(subject).to receive(:require_workers)
|
|
30
60
|
allow(subject).to receive(:validate_queues)
|
|
@@ -34,7 +64,7 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
|
34
64
|
|
|
35
65
|
specify do
|
|
36
66
|
Shoryuken.options[:queues] = ['queue1', 'queue2'] # default queues
|
|
37
|
-
Shoryuken.options[:groups] = [[
|
|
67
|
+
Shoryuken.options[:groups] = [['custom', { queues: ['queue3'], delay: 25 }]]
|
|
38
68
|
subject.load
|
|
39
69
|
|
|
40
70
|
expect(Shoryuken.groups['default'][:queues]).to eq(%w[queue1 queue2])
|
|
@@ -44,10 +74,9 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
|
44
74
|
end
|
|
45
75
|
end
|
|
46
76
|
|
|
47
|
-
|
|
48
77
|
describe '#prefix_active_job_queue_names' do
|
|
49
78
|
before do
|
|
50
|
-
allow(subject).to receive(:
|
|
79
|
+
allow(subject).to receive(:initialize_rails)
|
|
51
80
|
allow(subject).to receive(:require_workers)
|
|
52
81
|
allow(subject).to receive(:validate_queues)
|
|
53
82
|
allow(subject).to receive(:validate_workers)
|
|
@@ -57,11 +86,11 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
|
57
86
|
ActiveJob::Base.queue_name_delimiter = '_'
|
|
58
87
|
|
|
59
88
|
allow(Shoryuken).to receive(:active_job?).and_return(true)
|
|
60
|
-
end
|
|
61
89
|
|
|
62
|
-
specify do
|
|
63
90
|
Shoryuken.active_job_queue_name_prefixing = true
|
|
91
|
+
end
|
|
64
92
|
|
|
93
|
+
specify do
|
|
65
94
|
Shoryuken.options[:queues] = ['queue1', ['queue2', 2]]
|
|
66
95
|
|
|
67
96
|
Shoryuken.options[:groups] = {
|
|
@@ -73,10 +102,32 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
|
73
102
|
expect(Shoryuken.groups['default'][:queues]).to eq(%w[test_queue1 test_queue2 test_queue2])
|
|
74
103
|
expect(Shoryuken.groups['group1'][:queues]).to eq(%w[test_group1_queue1 test_group1_queue2])
|
|
75
104
|
end
|
|
105
|
+
|
|
106
|
+
it 'does not prefix url-based queues' do
|
|
107
|
+
Shoryuken.options[:queues] = ['https://example.com/test_queue1']
|
|
108
|
+
Shoryuken.options[:groups] = { 'group1' => { queues: ['https://example.com/test_group1_queue1'] } }
|
|
109
|
+
|
|
110
|
+
subject.load
|
|
111
|
+
|
|
112
|
+
expect(Shoryuken.groups['default'][:queues]).to(eq(['https://example.com/test_queue1']))
|
|
113
|
+
expect(Shoryuken.groups['group1'][:queues]).to(eq(['https://example.com/test_group1_queue1']))
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
it 'does not prefix arn-based queues' do
|
|
117
|
+
Shoryuken.options[:queues] = ['arn:aws:sqs:fake-region-1:1234:test_queue1']
|
|
118
|
+
Shoryuken.options[:groups] = { 'group1' => { queues: ['arn:aws:sqs:fake-region-1:1234:test_group1_queue1'] } }
|
|
119
|
+
|
|
120
|
+
subject.load
|
|
121
|
+
|
|
122
|
+
expect(Shoryuken.groups['default'][:queues]).to(eq(['arn:aws:sqs:fake-region-1:1234:test_queue1']))
|
|
123
|
+
expect(Shoryuken.groups['group1'][:queues]).to(eq(['arn:aws:sqs:fake-region-1:1234:test_group1_queue1']))
|
|
124
|
+
end
|
|
76
125
|
end
|
|
126
|
+
|
|
77
127
|
describe "#setup_options" do
|
|
78
|
-
let
|
|
79
|
-
let
|
|
128
|
+
let(:cli_queues) { { "queue1" => 10, "queue2" => 20 } }
|
|
129
|
+
let(:config_queues) { [["queue1", 8], ["queue2", 4]] }
|
|
130
|
+
|
|
80
131
|
context "when given queues through config and CLI" do
|
|
81
132
|
specify do
|
|
82
133
|
allow_any_instance_of(Shoryuken::EnvironmentLoader).to receive(:config_file_options).and_return({ queues: config_queues })
|
|
@@ -84,6 +135,7 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
|
84
135
|
expect(Shoryuken.options[:queues]).to eq(cli_queues)
|
|
85
136
|
end
|
|
86
137
|
end
|
|
138
|
+
|
|
87
139
|
context "when given queues through config only" do
|
|
88
140
|
specify do
|
|
89
141
|
allow_any_instance_of(Shoryuken::EnvironmentLoader).to receive(:config_file_options).and_return({ queues: config_queues })
|
|
@@ -91,6 +143,7 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
|
91
143
|
expect(Shoryuken.options[:queues]).to eq(config_queues)
|
|
92
144
|
end
|
|
93
145
|
end
|
|
146
|
+
|
|
94
147
|
context "when given queues through CLI only" do
|
|
95
148
|
specify do
|
|
96
149
|
Shoryuken::EnvironmentLoader.setup_options(queues: cli_queues)
|
|
@@ -10,6 +10,10 @@ RSpec.describe ActiveJob::QueueAdapters::ShoryukenConcurrentSendAdapter do
|
|
|
10
10
|
let(:error_handler) { -> {} }
|
|
11
11
|
let(:success_handler) { -> {} }
|
|
12
12
|
|
|
13
|
+
before do
|
|
14
|
+
allow(Concurrent).to receive(:global_io_executor).and_return(Concurrent::ImmediateExecutor.new)
|
|
15
|
+
end
|
|
16
|
+
|
|
13
17
|
subject { described_class.new(success_handler, error_handler) }
|
|
14
18
|
|
|
15
19
|
context 'when success' do
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
require 'active_job'
|
|
3
|
+
require 'shoryuken/extensions/active_job_extensions'
|
|
4
|
+
require 'shoryuken/extensions/active_job_adapter'
|
|
5
|
+
|
|
6
|
+
RSpec.describe ActiveJob::QueueAdapters::ShoryukenAdapter::JobWrapper do
|
|
7
|
+
subject { described_class.new }
|
|
8
|
+
|
|
9
|
+
describe '#perform' do
|
|
10
|
+
it 'sets executions to reflect approximate receive count' do
|
|
11
|
+
attributes = { 'ApproximateReceiveCount' => '42' }
|
|
12
|
+
sqs_msg = double Shoryuken::Message, attributes: attributes
|
|
13
|
+
job_hash = { 'arguments' => [1, 2, 3] }
|
|
14
|
+
job_hash_with_executions = { 'arguments' => [1, 2, 3], 'executions' => 41 }
|
|
15
|
+
expect(ActiveJob::Base).to receive(:execute).with(job_hash_with_executions)
|
|
16
|
+
|
|
17
|
+
subject.perform sqs_msg, job_hash
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
require 'shoryuken/launcher'
|
|
3
|
+
|
|
4
|
+
RSpec.describe Shoryuken::Launcher do
|
|
5
|
+
let(:executor) do
|
|
6
|
+
# We can't use Concurrent.global_io_executor in these tests since once you
|
|
7
|
+
# shut down a thread pool, you can't start it back up. Instead, we create
|
|
8
|
+
# one new thread pool executor for each spec. We use a new
|
|
9
|
+
# CachedThreadPool, since that most closely resembles
|
|
10
|
+
# Concurrent.global_io_executor
|
|
11
|
+
Concurrent::CachedThreadPool.new auto_terminate: true
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
let(:first_group_manager) { double(:first_group_manager, group: 'first_group') }
|
|
15
|
+
let(:second_group_manager) { double(:second_group_manager, group: 'second_group') }
|
|
16
|
+
let(:first_queue) { "launcher_spec_#{SecureRandom.uuid}" }
|
|
17
|
+
let(:second_queue) { "launcher_spec_#{SecureRandom.uuid}" }
|
|
18
|
+
|
|
19
|
+
before do
|
|
20
|
+
Shoryuken.add_group('first_group', 1)
|
|
21
|
+
Shoryuken.add_group('second_group', 1)
|
|
22
|
+
Shoryuken.add_queue(first_queue, 1, 'first_group')
|
|
23
|
+
Shoryuken.add_queue(second_queue, 1, 'second_group')
|
|
24
|
+
allow(Shoryuken).to receive(:launcher_executor).and_return(executor)
|
|
25
|
+
allow(Shoryuken::Manager).to receive(:new).with('first_group', any_args).and_return(first_group_manager)
|
|
26
|
+
allow(Shoryuken::Manager).to receive(:new).with('second_group', any_args).and_return(second_group_manager)
|
|
27
|
+
allow(first_group_manager).to receive(:running?).and_return(true)
|
|
28
|
+
allow(second_group_manager).to receive(:running?).and_return(true)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
describe '#healthy?' do
|
|
32
|
+
context 'when all groups have managers' do
|
|
33
|
+
context 'when all managers are running' do
|
|
34
|
+
it 'returns true' do
|
|
35
|
+
expect(subject.healthy?).to be true
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
context 'when one manager is not running' do
|
|
40
|
+
before do
|
|
41
|
+
allow(second_group_manager).to receive(:running?).and_return(false)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
it 'returns false' do
|
|
45
|
+
expect(subject.healthy?).to be false
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
context 'when all groups do not have managers' do
|
|
51
|
+
before do
|
|
52
|
+
allow(second_group_manager).to receive(:group).and_return('some_random_group')
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
it 'returns false' do
|
|
56
|
+
expect(subject.healthy?).to be false
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
@@ -15,7 +15,7 @@ RSpec.describe Shoryuken::Manager do
|
|
|
15
15
|
let(:concurrency) { 1 }
|
|
16
16
|
let(:executor) { Concurrent::ImmediateExecutor.new }
|
|
17
17
|
|
|
18
|
-
subject { Shoryuken::Manager.new(fetcher, polling_strategy, concurrency, executor) }
|
|
18
|
+
subject { Shoryuken::Manager.new('default', fetcher, polling_strategy, concurrency, executor) }
|
|
19
19
|
|
|
20
20
|
before do
|
|
21
21
|
allow(fetcher).to receive(:fetch).and_return([])
|
|
@@ -71,6 +71,13 @@ RSpec.describe Shoryuken::Manager do
|
|
|
71
71
|
|
|
72
72
|
expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
|
|
73
73
|
expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
|
|
74
|
+
expect(subject).to receive(:fire_event).with(:utilization_update,
|
|
75
|
+
false,
|
|
76
|
+
{
|
|
77
|
+
group: 'default',
|
|
78
|
+
busy_processors: 1,
|
|
79
|
+
max_processors: 1
|
|
80
|
+
})
|
|
74
81
|
expect(Shoryuken::Processor).to receive(:process).with(q, message)
|
|
75
82
|
expect(Shoryuken.logger).to receive(:info).never
|
|
76
83
|
|
|
@@ -99,6 +106,13 @@ RSpec.describe Shoryuken::Manager do
|
|
|
99
106
|
q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
|
|
100
107
|
|
|
101
108
|
expect(fetcher).to receive(:fetch).with(q, described_class::BATCH_LIMIT).and_return(messages)
|
|
109
|
+
expect(subject).to receive(:fire_event).with(:utilization_update,
|
|
110
|
+
false,
|
|
111
|
+
{
|
|
112
|
+
group: 'default',
|
|
113
|
+
busy_processors: 1,
|
|
114
|
+
max_processors: 1
|
|
115
|
+
})
|
|
102
116
|
expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
|
|
103
117
|
allow(subject).to receive(:batched_queue?).with(q).and_return(true)
|
|
104
118
|
expect(Shoryuken::Processor).to receive(:process).with(q, messages)
|
|
@@ -163,4 +177,26 @@ RSpec.describe Shoryuken::Manager do
|
|
|
163
177
|
end
|
|
164
178
|
end
|
|
165
179
|
end
|
|
180
|
+
|
|
181
|
+
describe '#running?' do
|
|
182
|
+
context 'when the executor is running' do
|
|
183
|
+
before do
|
|
184
|
+
allow(executor).to receive(:running?).and_return(true)
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
it 'returns true' do
|
|
188
|
+
expect(subject.running?).to be true
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
context 'when the executor is not running' do
|
|
193
|
+
before do
|
|
194
|
+
allow(executor).to receive(:running?).and_return(false)
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
it 'returns false' do
|
|
198
|
+
expect(subject.running?).to be false
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
end
|
|
166
202
|
end
|
|
@@ -106,12 +106,37 @@ RSpec.describe Shoryuken::Polling::WeightedRoundRobin do
|
|
|
106
106
|
end
|
|
107
107
|
|
|
108
108
|
describe '#message_processed' do
|
|
109
|
-
it 'removes paused queue
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
109
|
+
it 'removes delay from paused queue' do
|
|
110
|
+
queues << queue1
|
|
111
|
+
queues << queue2
|
|
112
|
+
|
|
113
|
+
expect(subject.next_queue).to eq(queue1)
|
|
114
|
+
subject.messages_found(queue1, 0) # pauses queue1
|
|
115
|
+
|
|
116
|
+
expect(subject.active_queues).to eq([[queue2, 1]])
|
|
117
|
+
|
|
118
|
+
subject.message_processed(queue1) # marks queue1 to be unpaused
|
|
119
|
+
|
|
120
|
+
expect(subject.next_queue).to eq(queue2) # implicitly unpauses queue1
|
|
121
|
+
expect(subject.active_queues).to eq([[queue1, 1], [queue2, 1]])
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
it 'preserves weight of queues when unpausing' do
|
|
125
|
+
queues << queue1
|
|
126
|
+
queues << queue1
|
|
127
|
+
queues << queue2
|
|
128
|
+
|
|
129
|
+
expect(subject.next_queue).to eq(queue1)
|
|
130
|
+
subject.messages_found(queue1, 1)
|
|
131
|
+
|
|
132
|
+
expect(subject.next_queue).to eq(queue2)
|
|
133
|
+
subject.messages_found(queue2, 0) # pauses queue2
|
|
134
|
+
|
|
135
|
+
expect(subject.active_queues).to eq([[queue1, 2]])
|
|
136
|
+
subject.message_processed(queue2) # marks queue2 to be unpaused
|
|
137
|
+
|
|
138
|
+
expect(subject.next_queue).to eq(queue1) # implicitly unpauses queue2
|
|
139
|
+
expect(subject.active_queues).to eq([[queue1, 2], [queue2, 1]])
|
|
115
140
|
end
|
|
116
141
|
end
|
|
117
142
|
end
|
|
@@ -80,7 +80,8 @@ RSpec.describe Shoryuken::Queue do
|
|
|
80
80
|
end
|
|
81
81
|
|
|
82
82
|
it 'deletes' do
|
|
83
|
-
expect(sqs).to receive(:delete_message_batch).with(entries: entries,
|
|
83
|
+
expect(sqs).to receive(:delete_message_batch).with({ entries: entries,
|
|
84
|
+
queue_url: queue_url }).and_return(double(failed: []))
|
|
84
85
|
|
|
85
86
|
subject.delete_messages(entries: entries)
|
|
86
87
|
end
|
|
@@ -91,7 +92,8 @@ RSpec.describe Shoryuken::Queue do
|
|
|
91
92
|
logger = double 'Logger'
|
|
92
93
|
|
|
93
94
|
expect(sqs).to(
|
|
94
|
-
receive(:delete_message_batch).with(entries: entries,
|
|
95
|
+
receive(:delete_message_batch).with({ entries: entries,
|
|
96
|
+
queue_url: queue_url }).and_return(double(failed: [failure]))
|
|
95
97
|
)
|
|
96
98
|
expect(subject).to receive(:logger).and_return(logger)
|
|
97
99
|
expect(logger).to receive(:error)
|
|
@@ -157,7 +159,8 @@ RSpec.describe Shoryuken::Queue do
|
|
|
157
159
|
it 'accepts SQS request parameters' do
|
|
158
160
|
# https://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#send_message_batch-instance_method
|
|
159
161
|
expect(sqs).to(
|
|
160
|
-
receive(:send_message_batch).with(hash_including(entries: [{ id: '0', message_body: 'msg1' },
|
|
162
|
+
receive(:send_message_batch).with(hash_including(entries: [{ id: '0', message_body: 'msg1' },
|
|
163
|
+
{ id: '1', message_body: 'msg2' }]))
|
|
161
164
|
)
|
|
162
165
|
|
|
163
166
|
subject.send_messages(entries: [{ id: '0', message_body: 'msg1' }, { id: '1', message_body: 'msg2' }])
|
|
@@ -286,7 +289,8 @@ RSpec.describe Shoryuken::Queue do
|
|
|
286
289
|
Shoryuken.cache_visibility_timeout = false
|
|
287
290
|
|
|
288
291
|
expect(sqs).to(
|
|
289
|
-
receive(:get_queue_attributes).with(queue_url: queue_url,
|
|
292
|
+
receive(:get_queue_attributes).with(queue_url: queue_url,
|
|
293
|
+
attribute_names: ['All']).and_return(attribute_response).exactly(3).times
|
|
290
294
|
)
|
|
291
295
|
expect(subject.visibility_timeout).to eq(30)
|
|
292
296
|
expect(subject.visibility_timeout).to eq(30)
|
|
@@ -299,7 +303,8 @@ RSpec.describe Shoryuken::Queue do
|
|
|
299
303
|
Shoryuken.cache_visibility_timeout = true
|
|
300
304
|
|
|
301
305
|
expect(sqs).to(
|
|
302
|
-
receive(:get_queue_attributes).with(queue_url: queue_url,
|
|
306
|
+
receive(:get_queue_attributes).with({ queue_url: queue_url,
|
|
307
|
+
attribute_names: ['All'] }).and_return(attribute_response).once
|
|
303
308
|
)
|
|
304
309
|
expect(subject.visibility_timeout).to eq(30)
|
|
305
310
|
expect(subject.visibility_timeout).to eq(30)
|
|
@@ -10,16 +10,16 @@ RSpec.describe Shoryuken::Worker::DefaultExecutor do
|
|
|
10
10
|
|
|
11
11
|
describe '.perform_in' do
|
|
12
12
|
it 'delays a message' do
|
|
13
|
-
expect(sqs_queue).to receive(:send_message).with(
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
13
|
+
expect(sqs_queue).to receive(:send_message).with({
|
|
14
|
+
message_attributes: {
|
|
15
|
+
'shoryuken_class' => {
|
|
16
|
+
string_value: TestWorker.to_s,
|
|
17
|
+
data_type: 'String'
|
|
18
|
+
}
|
|
19
|
+
},
|
|
20
|
+
message_body: 'message',
|
|
21
|
+
delay_seconds: 60
|
|
22
|
+
})
|
|
23
23
|
|
|
24
24
|
TestWorker.perform_in(60, 'message')
|
|
25
25
|
end
|
|
@@ -33,16 +33,16 @@ RSpec.describe Shoryuken::Worker::DefaultExecutor do
|
|
|
33
33
|
|
|
34
34
|
describe '.perform_at' do
|
|
35
35
|
it 'delays a message' do
|
|
36
|
-
expect(sqs_queue).to receive(:send_message).with(
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
36
|
+
expect(sqs_queue).to receive(:send_message).with({
|
|
37
|
+
message_attributes: {
|
|
38
|
+
'shoryuken_class' => {
|
|
39
|
+
string_value: TestWorker.to_s,
|
|
40
|
+
data_type: 'String'
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
message_body: 'message',
|
|
44
|
+
delay_seconds: 60
|
|
45
|
+
})
|
|
46
46
|
|
|
47
47
|
TestWorker.perform_in(Time.now + 60, 'message')
|
|
48
48
|
end
|
|
@@ -56,30 +56,30 @@ RSpec.describe Shoryuken::Worker::DefaultExecutor do
|
|
|
56
56
|
|
|
57
57
|
describe '.perform_async' do
|
|
58
58
|
it 'enqueues a message' do
|
|
59
|
-
expect(sqs_queue).to receive(:send_message).with(
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
59
|
+
expect(sqs_queue).to receive(:send_message).with({
|
|
60
|
+
message_attributes: {
|
|
61
|
+
'shoryuken_class' => {
|
|
62
|
+
string_value: TestWorker.to_s,
|
|
63
|
+
data_type: 'String'
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
message_body: 'message'
|
|
67
|
+
})
|
|
68
68
|
|
|
69
69
|
TestWorker.perform_async('message')
|
|
70
70
|
end
|
|
71
71
|
|
|
72
72
|
it 'enqueues a message with options' do
|
|
73
|
-
expect(sqs_queue).to receive(:send_message).with(
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
73
|
+
expect(sqs_queue).to receive(:send_message).with({
|
|
74
|
+
delay_seconds: 60,
|
|
75
|
+
message_attributes: {
|
|
76
|
+
'shoryuken_class' => {
|
|
77
|
+
string_value: TestWorker.to_s,
|
|
78
|
+
data_type: 'String'
|
|
79
|
+
}
|
|
80
|
+
},
|
|
81
|
+
message_body: 'delayed message'
|
|
82
|
+
})
|
|
83
83
|
|
|
84
84
|
TestWorker.perform_async('delayed message', delay_seconds: 60)
|
|
85
85
|
end
|
|
@@ -89,15 +89,15 @@ RSpec.describe Shoryuken::Worker::DefaultExecutor do
|
|
|
89
89
|
|
|
90
90
|
expect(Shoryuken::Client).to receive(:queues).with(new_queue).and_return(sqs_queue)
|
|
91
91
|
|
|
92
|
-
expect(sqs_queue).to receive(:send_message).with(
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
92
|
+
expect(sqs_queue).to receive(:send_message).with({
|
|
93
|
+
message_attributes: {
|
|
94
|
+
'shoryuken_class' => {
|
|
95
|
+
string_value: TestWorker.to_s,
|
|
96
|
+
data_type: 'String'
|
|
97
|
+
}
|
|
98
|
+
},
|
|
99
|
+
message_body: 'delayed message'
|
|
100
|
+
})
|
|
101
101
|
|
|
102
102
|
TestWorker.perform_async('delayed message', queue: new_queue)
|
|
103
103
|
end
|
data/spec/shoryuken_spec.rb
CHANGED
|
@@ -1,4 +1,13 @@
|
|
|
1
1
|
require 'spec_helper'
|
|
2
2
|
|
|
3
3
|
RSpec.describe Shoryuken do
|
|
4
|
+
describe '.healthy?' do
|
|
5
|
+
before do
|
|
6
|
+
allow(Shoryuken::Runner).to receive(:instance).and_return(double(:instance, healthy?: :some_result))
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
it 'delegates to the runner instance' do
|
|
10
|
+
expect(described_class.healthy?).to eq(:some_result)
|
|
11
|
+
end
|
|
12
|
+
end
|
|
4
13
|
end
|
data/spec/spec_helper.rb
CHANGED
|
@@ -32,9 +32,6 @@ class TestWorker
|
|
|
32
32
|
end
|
|
33
33
|
|
|
34
34
|
RSpec.configure do |config|
|
|
35
|
-
# TODO: Run these tests again on CI
|
|
36
|
-
config.filter_run_excluding slow: true
|
|
37
|
-
|
|
38
35
|
config.before do
|
|
39
36
|
Shoryuken::Client.class_variable_set :@@queues, {}
|
|
40
37
|
|
|
@@ -63,7 +60,6 @@ RSpec.configure do |config|
|
|
|
63
60
|
|
|
64
61
|
Shoryuken.cache_visibility_timeout = false
|
|
65
62
|
|
|
66
|
-
allow(Concurrent).to receive(:global_io_executor).and_return(Concurrent::ImmediateExecutor.new)
|
|
67
63
|
allow(Shoryuken).to receive(:active_job?).and_return(false)
|
|
68
64
|
end
|
|
69
65
|
end
|