shoryuken 3.0.6 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +90 -24
- data/.travis.yml +17 -5
- data/CHANGELOG.md +265 -62
- data/Gemfile +9 -1
- data/Gemfile.aws-sdk-core-v2 +13 -0
- data/README.md +19 -113
- data/Rakefile +1 -1
- data/bin/cli/base.rb +0 -3
- data/bin/cli/sqs.rb +42 -16
- data/bin/shoryuken +4 -9
- data/examples/bootstrap_queues.rb +3 -3
- data/examples/default_worker.rb +2 -2
- data/lib/shoryuken/body_parser.rb +27 -0
- data/lib/shoryuken/client.rb +6 -2
- data/lib/shoryuken/core_ext.rb +1 -1
- data/lib/shoryuken/default_worker_registry.rb +2 -2
- data/lib/shoryuken/environment_loader.rb +60 -24
- data/lib/shoryuken/extensions/active_job_adapter.rb +21 -11
- data/lib/shoryuken/fetcher.rb +58 -19
- data/lib/shoryuken/launcher.rb +70 -7
- data/lib/shoryuken/logging.rb +1 -6
- data/lib/shoryuken/manager.rb +50 -80
- data/lib/shoryuken/middleware/chain.rb +4 -0
- data/lib/shoryuken/middleware/server/active_record.rb +1 -1
- data/lib/shoryuken/middleware/server/auto_delete.rb +4 -9
- data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +6 -9
- data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +9 -3
- data/lib/shoryuken/middleware/server/timing.rb +12 -16
- data/lib/shoryuken/options.rb +225 -0
- data/lib/shoryuken/polling/base.rb +67 -0
- data/lib/shoryuken/polling/strict_priority.rb +77 -0
- data/lib/shoryuken/polling/weighted_round_robin.rb +66 -0
- data/lib/shoryuken/processor.rb +30 -39
- data/lib/shoryuken/queue.rb +41 -10
- data/lib/shoryuken/runner.rb +13 -17
- data/lib/shoryuken/util.rb +3 -3
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker/default_executor.rb +33 -0
- data/lib/shoryuken/worker/inline_executor.rb +37 -0
- data/lib/shoryuken/worker.rb +76 -31
- data/lib/shoryuken/worker_registry.rb +4 -4
- data/lib/shoryuken.rb +54 -173
- data/shoryuken.gemspec +6 -6
- data/spec/integration/launcher_spec.rb +14 -8
- data/spec/shoryuken/body_parser_spec.rb +89 -0
- data/spec/shoryuken/client_spec.rb +1 -1
- data/spec/shoryuken/core_ext_spec.rb +6 -6
- data/spec/shoryuken/default_worker_registry_spec.rb +2 -4
- data/spec/shoryuken/environment_loader_spec.rb +32 -12
- data/spec/shoryuken/extensions/active_job_adapter_spec.rb +64 -0
- data/spec/shoryuken/fetcher_spec.rb +101 -18
- data/spec/shoryuken/manager_spec.rb +54 -26
- data/spec/shoryuken/middleware/chain_spec.rb +17 -5
- data/spec/shoryuken/middleware/server/auto_delete_spec.rb +9 -7
- data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb +4 -4
- data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +6 -4
- data/spec/shoryuken/middleware/server/timing_spec.rb +5 -3
- data/spec/shoryuken/options_spec.rb +180 -0
- data/spec/shoryuken/{polling_spec.rb → polling/strict_priority_spec.rb} +2 -101
- data/spec/shoryuken/polling/weighted_round_robin_spec.rb +99 -0
- data/spec/shoryuken/processor_spec.rb +26 -127
- data/spec/shoryuken/queue_spec.rb +115 -41
- data/spec/shoryuken/runner_spec.rb +3 -4
- data/spec/shoryuken/util_spec.rb +24 -0
- data/spec/shoryuken/worker/default_executor_spec.rb +105 -0
- data/spec/shoryuken/worker/inline_executor_spec.rb +49 -0
- data/spec/shoryuken/worker_spec.rb +35 -96
- data/spec/shoryuken_spec.rb +0 -59
- data/spec/spec_helper.rb +14 -3
- data/test_workers/endless_interruptive_worker.rb +2 -2
- data/test_workers/endless_uninterruptive_worker.rb +4 -4
- metadata +31 -12
- data/lib/shoryuken/polling.rb +0 -204
|
@@ -12,7 +12,7 @@ RSpec.describe Shoryuken::Client do
|
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
it 'memoizes queues' do
|
|
15
|
-
sqs.stub_responses(:get_queue_url, { queue_url: queue_url },
|
|
15
|
+
sqs.stub_responses(:get_queue_url, { queue_url: queue_url }, queue_url: 'xyz')
|
|
16
16
|
|
|
17
17
|
expect(Shoryuken::Client.queues(queue_name).url).to eq queue_url
|
|
18
18
|
expect(Shoryuken::Client.queues(queue_name).url).to eq queue_url
|
|
@@ -10,7 +10,7 @@ RSpec.describe 'Core Extensions' do
|
|
|
10
10
|
|
|
11
11
|
describe '#symbolize_keys' do
|
|
12
12
|
it 'converts keys into symbols' do
|
|
13
|
-
expect({ :key1 => 'value1', 'key2' => 'value2' }.symbolize_keys).to eq(:
|
|
13
|
+
expect({ :key1 => 'value1', 'key2' => 'value2' }.symbolize_keys).to eq(key1: 'value1', key2: 'value2')
|
|
14
14
|
end
|
|
15
15
|
end
|
|
16
16
|
|
|
@@ -20,11 +20,11 @@ RSpec.describe 'Core Extensions' do
|
|
|
20
20
|
'key2' => 'value2',
|
|
21
21
|
'key3' => {
|
|
22
22
|
'key31' => { 'key311' => 'value311' },
|
|
23
|
-
'key32' => 'value32'
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
23
|
+
'key32' => 'value32'
|
|
24
|
+
} }.deep_symbolize_keys).to eq(key1: 'value1',
|
|
25
|
+
key2: 'value2',
|
|
26
|
+
key3: { key31: { key311: 'value311' },
|
|
27
|
+
key32: 'value32' })
|
|
28
28
|
end
|
|
29
29
|
end
|
|
30
30
|
end
|
|
@@ -43,12 +43,10 @@ RSpec.describe Shoryuken::DefaultWorkerRegistry do
|
|
|
43
43
|
end
|
|
44
44
|
|
|
45
45
|
describe 'a registry with workers is handling messages' do
|
|
46
|
-
def build_message(
|
|
46
|
+
def build_message(_queue, explicit_worker = nil)
|
|
47
47
|
attributes = {}
|
|
48
48
|
|
|
49
|
-
if explicit_worker
|
|
50
|
-
attributes['shoryuken_class'] = { string_value: explicit_worker.to_s, data_type: 'String' }
|
|
51
|
-
end
|
|
49
|
+
attributes['shoryuken_class'] = { string_value: explicit_worker.to_s, data_type: 'String' } if explicit_worker
|
|
52
50
|
|
|
53
51
|
double(Shoryuken::Message,
|
|
54
52
|
body: 'test',
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
require 'spec_helper'
|
|
2
|
+
require 'active_job'
|
|
2
3
|
|
|
3
|
-
# rubocop:disable Metrics/BlockLength
|
|
4
4
|
RSpec.describe Shoryuken::EnvironmentLoader do
|
|
5
5
|
subject { described_class.new({}) }
|
|
6
6
|
|
|
7
7
|
describe '#parse_queues' do
|
|
8
8
|
before do
|
|
9
|
-
|
|
10
|
-
allow(subject).to receive(:load_rails).with(anything)
|
|
9
|
+
allow(subject).to receive(:load_rails)
|
|
11
10
|
allow(subject).to receive(:prefix_active_job_queue_names)
|
|
12
11
|
allow(subject).to receive(:require_workers)
|
|
13
12
|
allow(subject).to receive(:validate_queues)
|
|
@@ -15,20 +14,41 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
|
15
14
|
allow(subject).to receive(:patch_deprecated_workers)
|
|
16
15
|
end
|
|
17
16
|
|
|
18
|
-
|
|
19
|
-
Shoryuken.options[:queues] = ['
|
|
17
|
+
specify do
|
|
18
|
+
Shoryuken.options[:queues] = ['queue1', ['queue2', 2]]
|
|
20
19
|
subject.load
|
|
21
20
|
|
|
22
|
-
expect(Shoryuken.queues).to eq(%w
|
|
21
|
+
expect(Shoryuken.groups['default'][:queues]).to eq(%w[queue1 queue2 queue2])
|
|
23
22
|
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
describe '#prefix_active_job_queue_names' do
|
|
26
|
+
before do
|
|
27
|
+
allow(subject).to receive(:load_rails)
|
|
28
|
+
allow(subject).to receive(:require_workers)
|
|
29
|
+
allow(subject).to receive(:validate_queues)
|
|
30
|
+
allow(subject).to receive(:validate_workers)
|
|
31
|
+
allow(subject).to receive(:patch_deprecated_workers)
|
|
24
32
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
33
|
+
ActiveJob::Base.queue_name_prefix = 'test'
|
|
34
|
+
ActiveJob::Base.queue_name_delimiter = '_'
|
|
35
|
+
|
|
36
|
+
allow(Shoryuken).to receive(:active_job?).and_return(true)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
specify do
|
|
40
|
+
Shoryuken.active_job_queue_name_prefixing = true
|
|
41
|
+
|
|
42
|
+
Shoryuken.options[:queues] = ['queue1', ['queue2', 2]]
|
|
43
|
+
|
|
44
|
+
Shoryuken.options[:groups] = {
|
|
45
|
+
'group1' => { queues: %w[group1_queue1 group1_queue2] }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
subject.load
|
|
29
49
|
|
|
30
|
-
|
|
31
|
-
|
|
50
|
+
expect(Shoryuken.groups['default'][:queues]).to eq(%w[test_queue1 test_queue2 test_queue2])
|
|
51
|
+
expect(Shoryuken.groups['group1'][:queues]).to eq(%w[test_group1_queue1 test_group1_queue2])
|
|
32
52
|
end
|
|
33
53
|
end
|
|
34
54
|
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
require 'spec_helper'
|
|
2
|
+
require 'active_job'
|
|
3
|
+
require 'shoryuken/extensions/active_job_adapter'
|
|
4
|
+
|
|
5
|
+
RSpec.describe ActiveJob::QueueAdapters::ShoryukenAdapter do
|
|
6
|
+
let(:job) { double 'Job', id: '123', queue_name: 'queue' }
|
|
7
|
+
let(:fifo) { false }
|
|
8
|
+
let(:queue) { double 'Queue', fifo?: fifo }
|
|
9
|
+
|
|
10
|
+
before do
|
|
11
|
+
allow(Shoryuken::Client).to receive(:queues).with(job.queue_name).and_return(queue)
|
|
12
|
+
allow(job).to receive(:serialize).and_return(
|
|
13
|
+
'job_class' => 'Worker',
|
|
14
|
+
'job_id' => job.id,
|
|
15
|
+
'queue_name' => job.queue_name,
|
|
16
|
+
'arguments' => nil,
|
|
17
|
+
'locale' => nil
|
|
18
|
+
)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
describe '#enqueue' do
|
|
22
|
+
specify do
|
|
23
|
+
expect(queue).to receive(:send_message) do |hash|
|
|
24
|
+
expect(hash[:message_deduplication_id]).to_not be
|
|
25
|
+
end
|
|
26
|
+
expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
|
|
27
|
+
|
|
28
|
+
subject.enqueue(job)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
context 'when fifo' do
|
|
32
|
+
let(:fifo) { true }
|
|
33
|
+
|
|
34
|
+
it 'does not include job_id in the deduplication_id' do
|
|
35
|
+
expect(queue).to receive(:send_message) do |hash|
|
|
36
|
+
message_deduplication_id = Digest::SHA256.hexdigest(JSON.dump(job.serialize.except('job_id')))
|
|
37
|
+
|
|
38
|
+
expect(hash[:message_deduplication_id]).to eq(message_deduplication_id)
|
|
39
|
+
end
|
|
40
|
+
expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
|
|
41
|
+
|
|
42
|
+
subject.enqueue(job)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
describe '#enqueue_at' do
|
|
48
|
+
specify do
|
|
49
|
+
delay = 1
|
|
50
|
+
|
|
51
|
+
expect(queue).to receive(:send_message) do |hash|
|
|
52
|
+
expect(hash[:message_deduplication_id]).to_not be
|
|
53
|
+
expect(hash[:delay_seconds]).to eq(delay)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
expect(Shoryuken).to receive(:register_worker).with(job.queue_name, described_class::JobWrapper)
|
|
57
|
+
|
|
58
|
+
# need to figure out what to require Time.current and N.minutes to remove the stub
|
|
59
|
+
allow(subject).to receive(:calculate_delay).and_return(delay)
|
|
60
|
+
|
|
61
|
+
subject.enqueue_at(job, nil)
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -2,36 +2,119 @@ require 'spec_helper'
|
|
|
2
2
|
require 'shoryuken/manager'
|
|
3
3
|
require 'shoryuken/fetcher'
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
let(:
|
|
5
|
+
# rubocop:disable Metrics/BlockLength
|
|
6
|
+
RSpec.describe Shoryuken::Fetcher do
|
|
7
|
+
let(:queue) { instance_double('Shoryuken::Queue', fifo?: false) }
|
|
8
|
+
let(:queue_name) { 'default' }
|
|
8
9
|
let(:queue_config) { Shoryuken::Polling::QueueConfiguration.new(queue_name, {}) }
|
|
10
|
+
let(:group) { 'default' }
|
|
9
11
|
|
|
10
12
|
let(:sqs_msg) do
|
|
11
|
-
double(
|
|
13
|
+
double(
|
|
14
|
+
Shoryuken::Message,
|
|
12
15
|
queue_url: queue_name,
|
|
13
16
|
body: 'test',
|
|
14
|
-
message_id: 'fc754df79cc24c4196ca5996a44b771e'
|
|
15
|
-
|
|
17
|
+
message_id: 'fc754df79cc24c4196ca5996a44b771e'
|
|
18
|
+
)
|
|
16
19
|
end
|
|
17
20
|
|
|
18
|
-
subject { described_class.new }
|
|
21
|
+
subject { described_class.new(group) }
|
|
19
22
|
|
|
20
23
|
describe '#fetch' do
|
|
21
|
-
|
|
24
|
+
let(:limit) { 1 }
|
|
25
|
+
|
|
26
|
+
specify do
|
|
22
27
|
expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
28
|
+
|
|
29
|
+
Shoryuken.sqs_client_receive_message_opts[group] = { wait_time_seconds: 10 }
|
|
30
|
+
|
|
31
|
+
expect(queue).to receive(:receive_messages).with(
|
|
32
|
+
wait_time_seconds: 10,
|
|
33
|
+
max_number_of_messages: limit,
|
|
34
|
+
message_attribute_names: ['All'],
|
|
35
|
+
attribute_names: ['All']
|
|
36
|
+
).and_return([])
|
|
37
|
+
|
|
38
|
+
subject.fetch(queue_config, limit)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
it 'logs debug only' do
|
|
42
|
+
# See https://github.com/phstc/shoryuken/issues/435
|
|
43
|
+
logger = double 'logger'
|
|
44
|
+
|
|
45
|
+
allow(subject).to receive(:logger).and_return(logger)
|
|
46
|
+
|
|
47
|
+
expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
|
48
|
+
|
|
49
|
+
expect(queue).to receive(:receive_messages).and_return([double('SQS Msg')])
|
|
50
|
+
|
|
51
|
+
expect(logger).to receive(:debug).exactly(3).times
|
|
52
|
+
expect(logger).to_not receive(:info)
|
|
53
|
+
|
|
54
|
+
subject.fetch(queue_config, limit)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
context 'when receive options per queue' do
|
|
58
|
+
let(:limit) { 5 }
|
|
59
|
+
|
|
60
|
+
specify do
|
|
61
|
+
expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
|
62
|
+
|
|
63
|
+
Shoryuken.sqs_client_receive_message_opts[queue_name] = { max_number_of_messages: 1 }
|
|
64
|
+
|
|
65
|
+
expect(queue).to receive(:receive_messages).with(
|
|
66
|
+
max_number_of_messages: 1,
|
|
67
|
+
message_attribute_names: ['All'],
|
|
68
|
+
attribute_names: ['All']
|
|
69
|
+
).and_return([])
|
|
70
|
+
|
|
71
|
+
subject.fetch(queue_config, limit)
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
context 'when max_number_of_messages opt is great than limit' do
|
|
76
|
+
it 'uses limit' do
|
|
77
|
+
expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
|
78
|
+
|
|
79
|
+
Shoryuken.sqs_client_receive_message_opts[queue_name] = { max_number_of_messages: 20 }
|
|
80
|
+
|
|
81
|
+
expect(queue).to receive(:receive_messages).with(
|
|
82
|
+
max_number_of_messages: limit,
|
|
83
|
+
message_attribute_names: ['All'],
|
|
84
|
+
attribute_names: ['All']
|
|
85
|
+
).and_return([])
|
|
86
|
+
|
|
87
|
+
subject.fetch(queue_config, limit)
|
|
88
|
+
end
|
|
27
89
|
end
|
|
28
90
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
and_return(
|
|
34
|
-
|
|
91
|
+
context 'when limit is greater than FETCH_LIMIT' do
|
|
92
|
+
let(:limit) { 20 }
|
|
93
|
+
|
|
94
|
+
specify do
|
|
95
|
+
allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
|
96
|
+
expect(queue).to receive(:receive_messages).with(
|
|
97
|
+
max_number_of_messages: described_class::FETCH_LIMIT, attribute_names: ['All'], message_attribute_names: ['All']
|
|
98
|
+
).and_return([])
|
|
99
|
+
|
|
100
|
+
subject.fetch(queue_config, limit)
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
context 'when FIFO' do
|
|
105
|
+
let(:limit) { 10 }
|
|
106
|
+
let(:queue) { instance_double('Shoryuken::Queue', fifo?: true) }
|
|
107
|
+
|
|
108
|
+
it 'polls one message at the time' do
|
|
109
|
+
# see https://github.com/phstc/shoryuken/pull/530
|
|
110
|
+
|
|
111
|
+
allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
|
112
|
+
expect(queue).to receive(:receive_messages).with(
|
|
113
|
+
max_number_of_messages: 1, attribute_names: ['All'], message_attribute_names: ['All']
|
|
114
|
+
).and_return([])
|
|
115
|
+
|
|
116
|
+
subject.fetch(queue_config, limit)
|
|
117
|
+
end
|
|
35
118
|
end
|
|
36
119
|
end
|
|
37
120
|
end
|
|
@@ -11,57 +11,85 @@ RSpec.describe Shoryuken::Manager do
|
|
|
11
11
|
let(:queue) { 'default' }
|
|
12
12
|
let(:queues) { [queue] }
|
|
13
13
|
let(:polling_strategy) { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
|
|
14
|
-
let(:fetcher) { Shoryuken::Fetcher
|
|
14
|
+
let(:fetcher) { double Shoryuken::Fetcher }
|
|
15
15
|
let(:concurrency) { 1 }
|
|
16
|
+
let(:executor) { Concurrent::ImmediateExecutor.new }
|
|
16
17
|
|
|
17
|
-
subject { Shoryuken::Manager.new(fetcher, polling_strategy) }
|
|
18
|
+
subject { Shoryuken::Manager.new(fetcher, polling_strategy, concurrency, executor) }
|
|
18
19
|
|
|
19
|
-
before
|
|
20
|
-
|
|
20
|
+
before do
|
|
21
|
+
allow(fetcher).to receive(:fetch).and_return([])
|
|
21
22
|
end
|
|
22
23
|
|
|
23
|
-
after
|
|
24
|
+
after do
|
|
24
25
|
Shoryuken.options[:concurrency] = 1
|
|
25
26
|
TestWorker.get_shoryuken_options['batch'] = false
|
|
26
27
|
end
|
|
27
28
|
|
|
28
|
-
describe '
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
expect
|
|
32
|
-
|
|
29
|
+
describe '#stop' do
|
|
30
|
+
specify do
|
|
31
|
+
allow(subject).to receive(:running?).and_return(true, true, false)
|
|
32
|
+
expect(subject).to receive(:dispatch).once.and_call_original
|
|
33
|
+
expect(subject).to receive(:dispatch_loop).twice.and_call_original
|
|
34
|
+
subject.start
|
|
33
35
|
end
|
|
34
36
|
end
|
|
35
37
|
|
|
36
38
|
describe '#start' do
|
|
37
|
-
|
|
39
|
+
before do
|
|
40
|
+
# prevent dispatch loop
|
|
41
|
+
allow(subject).to receive(:running?).and_return(true, true, false)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
it 'pauses when there are no active queues' do
|
|
38
45
|
expect(polling_strategy).to receive(:next_queue).and_return(nil)
|
|
39
|
-
|
|
46
|
+
expect(subject).to receive(:dispatch).and_call_original
|
|
40
47
|
subject.start
|
|
41
48
|
end
|
|
42
49
|
|
|
43
|
-
|
|
50
|
+
it 'calls dispatch_batch if worker wants batches' do
|
|
44
51
|
TestWorker.get_shoryuken_options['batch'] = true
|
|
45
|
-
|
|
46
|
-
expect(subject).to receive(:dispatch_later)
|
|
52
|
+
expect(subject).to receive(:dispatch_batch).with(queue_config_of(queue))
|
|
47
53
|
subject.start
|
|
48
54
|
end
|
|
49
55
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
with(queue_config_of(queue))
|
|
53
|
-
expect(subject).to receive(:dispatch_later)
|
|
56
|
+
it 'calls dispatch_single_messages if worker wants single messages' do
|
|
57
|
+
expect(subject).to receive(:dispatch_single_messages).with(queue_config_of(queue))
|
|
54
58
|
subject.start
|
|
55
59
|
end
|
|
56
60
|
end
|
|
57
61
|
|
|
58
|
-
describe '#
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
62
|
+
describe '#dispatch' do
|
|
63
|
+
before do
|
|
64
|
+
allow(subject).to receive(:running?).and_return(true, true, false)
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
specify do
|
|
68
|
+
message = ['test1']
|
|
69
|
+
messages = [message]
|
|
70
|
+
q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
|
|
71
|
+
|
|
72
|
+
expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
|
|
73
|
+
expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
|
|
74
|
+
expect(Shoryuken::Processor).to receive(:process).with(q, message)
|
|
75
|
+
expect(Shoryuken.logger).to_not receive(:info)
|
|
76
|
+
|
|
77
|
+
subject.send(:dispatch)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
context 'when batch' do
|
|
81
|
+
specify do
|
|
82
|
+
messages = %w[test1 test2 test3]
|
|
83
|
+
q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
|
|
84
|
+
|
|
85
|
+
expect(fetcher).to receive(:fetch).with(q, described_class::BATCH_LIMIT).and_return(messages)
|
|
86
|
+
expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
|
|
87
|
+
allow(subject).to receive(:batched_queue?).with(q).and_return(true)
|
|
88
|
+
expect(Shoryuken::Processor).to receive(:process).with(q, messages)
|
|
89
|
+
expect(Shoryuken.logger).to_not receive(:info)
|
|
90
|
+
|
|
91
|
+
subject.send(:dispatch)
|
|
92
|
+
end
|
|
65
93
|
end
|
|
66
94
|
end
|
|
67
95
|
|
|
@@ -1,33 +1,45 @@
|
|
|
1
1
|
require 'spec_helper'
|
|
2
2
|
|
|
3
|
-
describe Shoryuken::Middleware::Chain do
|
|
3
|
+
RSpec.describe Shoryuken::Middleware::Chain do
|
|
4
4
|
class CustomMiddleware
|
|
5
5
|
def initialize(name, recorder)
|
|
6
6
|
@name = name
|
|
7
7
|
@recorder = recorder
|
|
8
8
|
end
|
|
9
9
|
|
|
10
|
-
def call(*
|
|
10
|
+
def call(*_args)
|
|
11
11
|
@recorder << [@name, 'before']
|
|
12
12
|
yield
|
|
13
13
|
@recorder << [@name, 'after']
|
|
14
14
|
end
|
|
15
15
|
end
|
|
16
16
|
|
|
17
|
+
class CustomMiddlewareB < CustomMiddleware; end
|
|
18
|
+
|
|
17
19
|
it 'supports custom middleware' do
|
|
18
20
|
subject.add CustomMiddleware, 1, []
|
|
19
21
|
|
|
20
22
|
expect(CustomMiddleware).to eq subject.entries.last.klass
|
|
21
23
|
end
|
|
22
24
|
|
|
25
|
+
it 'can add middleware to the front of chain' do
|
|
26
|
+
subject.prepend CustomMiddleware, 1, []
|
|
27
|
+
|
|
28
|
+
expect([CustomMiddleware]).to eq subject.entries.map(&:klass)
|
|
29
|
+
|
|
30
|
+
subject.prepend CustomMiddlewareB, 1, []
|
|
31
|
+
|
|
32
|
+
expect([CustomMiddlewareB, CustomMiddleware]).to eq subject.entries.map(&:klass)
|
|
33
|
+
end
|
|
34
|
+
|
|
23
35
|
it 'invokes a middleware' do
|
|
24
36
|
recorder = []
|
|
25
|
-
subject.add CustomMiddleware, '
|
|
37
|
+
subject.add CustomMiddleware, 'custom', recorder
|
|
26
38
|
|
|
27
39
|
final_action = nil
|
|
28
40
|
subject.invoke { final_action = true }
|
|
29
41
|
expect(final_action).to eq true
|
|
30
|
-
expect(recorder).to eq [%w[
|
|
42
|
+
expect(recorder).to eq [%w[custom before], %w[custom after]]
|
|
31
43
|
end
|
|
32
44
|
|
|
33
45
|
class NonYieldingMiddleware
|
|
@@ -37,7 +49,7 @@ describe Shoryuken::Middleware::Chain do
|
|
|
37
49
|
it 'allows middleware to abruptly stop processing rest of chain' do
|
|
38
50
|
recorder = []
|
|
39
51
|
subject.add NonYieldingMiddleware
|
|
40
|
-
subject.add CustomMiddleware, '
|
|
52
|
+
subject.add CustomMiddleware, 'custom', recorder
|
|
41
53
|
|
|
42
54
|
final_action = nil
|
|
43
55
|
subject.invoke { final_action = true }
|
|
@@ -6,9 +6,9 @@ describe Shoryuken::Middleware::Server::AutoDelete do
|
|
|
6
6
|
|
|
7
7
|
def build_message
|
|
8
8
|
double Shoryuken::Message,
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
queue_url: queue,
|
|
10
|
+
body: 'test',
|
|
11
|
+
receipt_handle: SecureRandom.uuid
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
let(:sqs_msg) { build_message }
|
|
@@ -21,7 +21,8 @@ describe Shoryuken::Middleware::Server::AutoDelete do
|
|
|
21
21
|
TestWorker.get_shoryuken_options['auto_delete'] = true
|
|
22
22
|
|
|
23
23
|
expect(sqs_queue).to receive(:delete_messages).with(entries: [
|
|
24
|
-
|
|
24
|
+
{ id: '0', receipt_handle: sqs_msg.receipt_handle }
|
|
25
|
+
])
|
|
25
26
|
|
|
26
27
|
subject.call(TestWorker.new, queue, sqs_msg, sqs_msg.body) {}
|
|
27
28
|
end
|
|
@@ -35,9 +36,10 @@ describe Shoryuken::Middleware::Server::AutoDelete do
|
|
|
35
36
|
sqs_msgs = [sqs_msg, sqs_msg2, sqs_msg3]
|
|
36
37
|
|
|
37
38
|
expect(sqs_queue).to receive(:delete_messages).with(entries: [
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
39
|
+
{ id: '0', receipt_handle: sqs_msg.receipt_handle },
|
|
40
|
+
{ id: '1', receipt_handle: sqs_msg2.receipt_handle },
|
|
41
|
+
{ id: '2', receipt_handle: sqs_msg3.receipt_handle }
|
|
42
|
+
])
|
|
41
43
|
|
|
42
44
|
subject.call(TestWorker.new, queue, sqs_msgs, [sqs_msg.body, sqs_msg2.body, sqs_msg3.body]) {}
|
|
43
45
|
end
|
|
@@ -18,9 +18,9 @@ RSpec.describe Shoryuken::Middleware::Server::AutoExtendVisibility do
|
|
|
18
18
|
end
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
-
def run_and_raise(worker, queue, sqs_msg
|
|
21
|
+
def run_and_raise(worker, queue, sqs_msg)
|
|
22
22
|
Shoryuken::Middleware::Server::AutoExtendVisibility.new.call(worker, queue, sqs_msg, sqs_msg.body) do
|
|
23
|
-
raise
|
|
23
|
+
raise
|
|
24
24
|
end
|
|
25
25
|
end
|
|
26
26
|
end
|
|
@@ -34,7 +34,7 @@ RSpec.describe Shoryuken::Middleware::Server::AutoExtendVisibility do
|
|
|
34
34
|
|
|
35
35
|
context 'when batch worker' do
|
|
36
36
|
it 'yields' do
|
|
37
|
-
expect { |b| subject.call(
|
|
37
|
+
expect { |b| subject.call(TestWorker.new, nil, [], nil, &b) }.to yield_control
|
|
38
38
|
end
|
|
39
39
|
end
|
|
40
40
|
|
|
@@ -53,7 +53,7 @@ RSpec.describe Shoryuken::Middleware::Server::AutoExtendVisibility do
|
|
|
53
53
|
allow(sqs_msg).to receive(:queue) { sqs_queue }
|
|
54
54
|
expect(sqs_msg).to_not receive(:change_visibility)
|
|
55
55
|
|
|
56
|
-
expect { Runner.new.run_and_raise(TestWorker.new, queue, sqs_msg
|
|
56
|
+
expect { Runner.new.run_and_raise(TestWorker.new, queue, sqs_msg) }.to raise_error
|
|
57
57
|
end
|
|
58
58
|
|
|
59
59
|
it 'does not extend message visibility if auto_visibility_timeout is not true' do
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
require 'spec_helper'
|
|
2
2
|
|
|
3
|
-
# rubocop:disable
|
|
3
|
+
# rubocop:disable /BlockLength, Metrics/
|
|
4
4
|
RSpec.describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
|
|
5
5
|
let(:queue) { 'default' }
|
|
6
6
|
let(:sqs_queue) { double Shoryuken::Queue }
|
|
7
|
-
let(:sqs_msg) {
|
|
8
|
-
|
|
7
|
+
let(:sqs_msg) {
|
|
8
|
+
double Shoryuken::Message, queue_url: queue, body: 'test', receipt_handle: SecureRandom.uuid,
|
|
9
|
+
attributes: { 'ApproximateReceiveCount' => 1 }, message_id: SecureRandom.uuid
|
|
10
|
+
}
|
|
9
11
|
|
|
10
12
|
before do
|
|
11
13
|
allow(Shoryuken::Client).to receive(:queues).with(queue).and_return(sqs_queue)
|
|
@@ -13,7 +15,7 @@ RSpec.describe Shoryuken::Middleware::Server::ExponentialBackoffRetry do
|
|
|
13
15
|
|
|
14
16
|
context 'when batch worker' do
|
|
15
17
|
it 'yields' do
|
|
16
|
-
expect { |b| subject.call(
|
|
18
|
+
expect { |b| subject.call(TestWorker.new, nil, [], nil, &b) }.to yield_control
|
|
17
19
|
end
|
|
18
20
|
end
|
|
19
21
|
|
|
@@ -1,14 +1,16 @@
|
|
|
1
1
|
require 'spec_helper'
|
|
2
2
|
|
|
3
|
-
describe Shoryuken::Middleware::Server::Timing do
|
|
3
|
+
RSpec.describe Shoryuken::Middleware::Server::Timing do
|
|
4
4
|
let(:queue) { 'default' }
|
|
5
5
|
let(:sqs_queue) { double Shoryuken::Queue, visibility_timeout: 60 }
|
|
6
6
|
|
|
7
7
|
let(:sqs_msg) do
|
|
8
|
-
double
|
|
8
|
+
double(
|
|
9
|
+
Shoryuken::Message,
|
|
9
10
|
queue_url: queue,
|
|
10
11
|
body: 'test',
|
|
11
12
|
message_id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e'
|
|
13
|
+
)
|
|
12
14
|
end
|
|
13
15
|
|
|
14
16
|
before do
|
|
@@ -28,7 +30,7 @@ describe Shoryuken::Middleware::Server::Timing do
|
|
|
28
30
|
|
|
29
31
|
context 'when exceeded the `visibility_timeout`' do
|
|
30
32
|
it 'logs exceeded' do
|
|
31
|
-
allow(subject).to receive(:elapsed).and_return(
|
|
33
|
+
allow(subject).to receive(:elapsed).and_return(120_000)
|
|
32
34
|
|
|
33
35
|
expect(Shoryuken.logger).to receive(:info) do |&block|
|
|
34
36
|
expect(block.call).to match(/started at/)
|