shoryuken 3.1.6 → 3.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +1 -1
- data/CHANGELOG.md +11 -0
- data/bin/cli/sqs.rb +8 -4
- data/bin/shoryuken +1 -1
- data/lib/shoryuken/fetcher.rb +13 -7
- data/lib/shoryuken/options.rb +12 -1
- data/lib/shoryuken/runner.rb +3 -1
- data/lib/shoryuken/version.rb +1 -1
- data/spec/shoryuken/fetcher_spec.rb +45 -9
- data/spec/shoryuken/options_spec.rb +80 -0
- data/spec/spec_helper.rb +2 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 52a8705f1b474b12c77323776c903bd2e1ff6ba1
|
4
|
+
data.tar.gz: 25792ada1e4231fa80c7c461affa364bbe8f9afa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bcc31cc6ea76c07965f0ef5e4d13f29407fcf7423d8441df46867255cd5beaec0e694f9d5e45f6be181d1f1f926f51752f64d6027be8dfacfbd721052138de70
|
7
|
+
data.tar.gz: 4c7a4b04713c37edbf2e1e2965b3135a6d603ff686f42f74a45f6e5b1b9212f71db155703c92433785357281568ec705cd02e23413abcb46dc079b43df57bad7
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,14 @@
|
|
1
|
+
## [v3.1.7] - 2017-07-31
|
2
|
+
|
3
|
+
- Allow polling strategy per group
|
4
|
+
- [#417](https://github.com/phstc/shoryuken/pull/417)
|
5
|
+
|
6
|
+
- Add support for creating FIFO queues
|
7
|
+
- [#419](https://github.com/phstc/shoryuken/pull/419)
|
8
|
+
|
9
|
+
- Allow receive message options per queue
|
10
|
+
- [#420](https://github.com/phstc/shoryuken/pull/420)
|
11
|
+
|
1
12
|
## [v3.1.6] - 2017-07-24
|
2
13
|
|
3
14
|
- Fix issue with dispatch_loop and delays
|
data/bin/cli/sqs.rb
CHANGED
@@ -96,8 +96,8 @@ module Shoryuken
|
|
96
96
|
end
|
97
97
|
|
98
98
|
desc 'ls [QUEUE-NAME-PREFIX]', 'Lists queues'
|
99
|
-
method_option :watch,
|
100
|
-
method_option :
|
99
|
+
method_option :watch, aliases: '-w', type: :boolean, desc: 'watch queues'
|
100
|
+
method_option :interval, aliases: '-n', type: :numeric, default: 2, desc: 'watch interval in seconds'
|
101
101
|
def ls(queue_name_prefix = '')
|
102
102
|
trap('SIGINT', 'EXIT') # expect ctrl-c from loop
|
103
103
|
|
@@ -108,7 +108,7 @@ module Shoryuken
|
|
108
108
|
|
109
109
|
break unless options[:watch]
|
110
110
|
|
111
|
-
sleep options[:
|
111
|
+
sleep options[:interval]
|
112
112
|
puts
|
113
113
|
end
|
114
114
|
end
|
@@ -187,8 +187,12 @@ module Shoryuken
|
|
187
187
|
end
|
188
188
|
|
189
189
|
desc 'create QUEUE-NAME', 'Create a queue'
|
190
|
+
method_option :attributes, aliases: '-a', type: :hash, default: {}, desc: 'queue attributes'
|
190
191
|
def create(queue_name)
|
191
|
-
|
192
|
+
attributes = options[:attributes]
|
193
|
+
attributes['FifoQueue'] ||= 'true' if queue_name.end_with?('.fifo')
|
194
|
+
|
195
|
+
queue_url = sqs.create_queue(queue_name: queue_name, attributes: attributes).queue_url
|
192
196
|
|
193
197
|
say "Queue #{queue_name} was successfully created. Queue URL #{queue_url}", :green
|
194
198
|
end
|
data/bin/shoryuken
CHANGED
@@ -32,7 +32,7 @@ module Shoryuken
|
|
32
32
|
opts = options.to_h.symbolize_keys
|
33
33
|
|
34
34
|
if opts[:config_file]
|
35
|
-
say
|
35
|
+
say '[DEPRECATED] Please use --config instead of --config-file', :yellow
|
36
36
|
end
|
37
37
|
|
38
38
|
opts[:config_file] = opts.delete(:config) if opts[:config]
|
data/lib/shoryuken/fetcher.rb
CHANGED
@@ -4,8 +4,6 @@ module Shoryuken
|
|
4
4
|
|
5
5
|
FETCH_LIMIT = 10
|
6
6
|
|
7
|
-
attr_reader :group
|
8
|
-
|
9
7
|
def initialize(group)
|
10
8
|
@group = group
|
11
9
|
end
|
@@ -26,12 +24,9 @@ module Shoryuken
|
|
26
24
|
private
|
27
25
|
|
28
26
|
def receive_messages(queue, limit)
|
29
|
-
|
30
|
-
limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
|
31
|
-
|
32
|
-
options = Shoryuken.sqs_client_receive_message_opts[group].to_h.dup
|
27
|
+
options = receive_options(queue)
|
33
28
|
|
34
|
-
options[:max_number_of_messages] = limit
|
29
|
+
options[:max_number_of_messages] = max_number_of_messages(limit, options)
|
35
30
|
options[:message_attribute_names] = %w(All)
|
36
31
|
options[:attribute_names] = %w(All)
|
37
32
|
|
@@ -39,5 +34,16 @@ module Shoryuken
|
|
39
34
|
|
40
35
|
Shoryuken::Client.queues(queue.name).receive_messages(options)
|
41
36
|
end
|
37
|
+
|
38
|
+
def max_number_of_messages(limit, options)
|
39
|
+
[limit, FETCH_LIMIT, options[:max_number_of_messages]].compact.min
|
40
|
+
end
|
41
|
+
|
42
|
+
def receive_options(queue)
|
43
|
+
options = Shoryuken.sqs_client_receive_message_opts[queue.name]
|
44
|
+
options ||= Shoryuken.sqs_client_receive_message_opts[@group]
|
45
|
+
|
46
|
+
options.to_h.dup
|
47
|
+
end
|
42
48
|
end
|
43
49
|
end
|
data/lib/shoryuken/options.rb
CHANGED
@@ -57,7 +57,18 @@ module Shoryuken
|
|
57
57
|
end
|
58
58
|
|
59
59
|
def polling_strategy(group)
|
60
|
-
options[group].to_h
|
60
|
+
strategy = (group == 'default' ? options : options[:groups][group]).to_h[:polling_strategy]
|
61
|
+
|
62
|
+
case strategy
|
63
|
+
when 'WeightedRoundRobin', nil # Default case
|
64
|
+
Polling::WeightedRoundRobin
|
65
|
+
when 'StrictPriority'
|
66
|
+
Polling::StrictPriority
|
67
|
+
when Class
|
68
|
+
strategy
|
69
|
+
else
|
70
|
+
raise ArgumentError, "#{strategy} is not a valid polling_strategy"
|
71
|
+
end
|
61
72
|
end
|
62
73
|
|
63
74
|
def start_callback
|
data/lib/shoryuken/runner.rb
CHANGED
@@ -118,10 +118,12 @@ module Shoryuken
|
|
118
118
|
end
|
119
119
|
|
120
120
|
def handle_signal(sig)
|
121
|
+
logger.debug "Got #{sig} signal"
|
122
|
+
|
121
123
|
case sig
|
122
124
|
when 'USR1' then execute_soft_shutdown
|
123
125
|
when 'TTIN' then print_threads_backtrace
|
124
|
-
|
126
|
+
when 'TERM', 'INT'
|
125
127
|
logger.info { "Received #{sig}, will shutdown down" }
|
126
128
|
|
127
129
|
raise Interrupt
|
data/lib/shoryuken/version.rb
CHANGED
@@ -2,6 +2,7 @@ require 'spec_helper'
|
|
2
2
|
require 'shoryuken/manager'
|
3
3
|
require 'shoryuken/fetcher'
|
4
4
|
|
5
|
+
# rubocop:disable Metrics/BlockLength
|
5
6
|
RSpec.describe Shoryuken::Fetcher do
|
6
7
|
let(:queue) { instance_double('Shoryuken::Queue') }
|
7
8
|
let(:queue_name) { 'default' }
|
@@ -13,7 +14,7 @@ RSpec.describe Shoryuken::Fetcher do
|
|
13
14
|
Shoryuken::Message,
|
14
15
|
queue_url: queue_name,
|
15
16
|
body: 'test',
|
16
|
-
message_id: 'fc754df79cc24c4196ca5996a44b771e'
|
17
|
+
message_id: 'fc754df79cc24c4196ca5996a44b771e'
|
17
18
|
)
|
18
19
|
end
|
19
20
|
|
@@ -27,23 +28,58 @@ RSpec.describe Shoryuken::Fetcher do
|
|
27
28
|
|
28
29
|
Shoryuken.sqs_client_receive_message_opts[group] = { wait_time_seconds: 10 }
|
29
30
|
|
30
|
-
expect(queue).to receive(:receive_messages).
|
31
|
-
|
32
|
-
|
31
|
+
expect(queue).to receive(:receive_messages).with(
|
32
|
+
wait_time_seconds: 10,
|
33
|
+
max_number_of_messages: limit,
|
34
|
+
message_attribute_names: ['All'],
|
35
|
+
attribute_names: ['All']
|
36
|
+
).and_return([])
|
33
37
|
|
34
38
|
subject.fetch(queue_config, limit)
|
35
39
|
end
|
36
40
|
|
41
|
+
context 'when receive options per queue' do
|
42
|
+
let(:limit) { 5 }
|
43
|
+
|
44
|
+
specify do
|
45
|
+
expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
46
|
+
|
47
|
+
Shoryuken.sqs_client_receive_message_opts[queue_name] = { max_number_of_messages: 1 }
|
48
|
+
|
49
|
+
expect(queue).to receive(:receive_messages).with(
|
50
|
+
max_number_of_messages: 1,
|
51
|
+
message_attribute_names: ['All'],
|
52
|
+
attribute_names: ['All']
|
53
|
+
).and_return([])
|
54
|
+
|
55
|
+
subject.fetch(queue_config, limit)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
context 'when max_number_of_messages opt is great than limit' do
|
60
|
+
it 'uses limit' do
|
61
|
+
expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
62
|
+
|
63
|
+
Shoryuken.sqs_client_receive_message_opts[queue_name] = { max_number_of_messages: 20 }
|
64
|
+
|
65
|
+
expect(queue).to receive(:receive_messages).with(
|
66
|
+
max_number_of_messages: limit,
|
67
|
+
message_attribute_names: ['All'],
|
68
|
+
attribute_names: ['All']
|
69
|
+
).and_return([])
|
70
|
+
|
71
|
+
subject.fetch(queue_config, limit)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
37
75
|
context 'when limit is greater than FETCH_LIMIT' do
|
38
76
|
let(:limit) { 20 }
|
39
77
|
|
40
78
|
specify do
|
41
|
-
Shoryuken.sqs_client_receive_message_opts[group] = {}
|
42
|
-
|
43
79
|
allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
44
|
-
expect(queue).to receive(:receive_messages).
|
45
|
-
|
46
|
-
|
80
|
+
expect(queue).to receive(:receive_messages).with(
|
81
|
+
max_number_of_messages: described_class::FETCH_LIMIT, attribute_names: ['All'], message_attribute_names: ['All']
|
82
|
+
).and_return([])
|
47
83
|
|
48
84
|
subject.fetch(queue_config, limit)
|
49
85
|
end
|
@@ -97,4 +97,84 @@ RSpec.describe Shoryuken::Options do
|
|
97
97
|
"and Shoryuken doesn't support a batchable worker for a queue with multiple workers")
|
98
98
|
end
|
99
99
|
end
|
100
|
+
|
101
|
+
describe '.polling_strategy' do
|
102
|
+
context 'when not set' do
|
103
|
+
specify do
|
104
|
+
expect(Shoryuken.polling_strategy('default')).to eq Shoryuken::Polling::WeightedRoundRobin
|
105
|
+
expect(Shoryuken.polling_strategy('group1')).to eq Shoryuken::Polling::WeightedRoundRobin
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
context 'when set to StrictPriority string' do
|
110
|
+
before do
|
111
|
+
Shoryuken.options[:polling_strategy] = 'StrictPriority'
|
112
|
+
|
113
|
+
Shoryuken.options[:groups] = {
|
114
|
+
'group1' => {
|
115
|
+
polling_strategy: 'StrictPriority'
|
116
|
+
}
|
117
|
+
}
|
118
|
+
end
|
119
|
+
|
120
|
+
specify do
|
121
|
+
expect(Shoryuken.polling_strategy('default')).to eq Shoryuken::Polling::StrictPriority
|
122
|
+
expect(Shoryuken.polling_strategy('group1')).to eq Shoryuken::Polling::StrictPriority
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
context 'when set to WeightedRoundRobin string' do
|
127
|
+
before do
|
128
|
+
Shoryuken.options[:polling_strategy] = 'WeightedRoundRobin'
|
129
|
+
|
130
|
+
Shoryuken.options[:groups] = {
|
131
|
+
'group1' => {
|
132
|
+
polling_strategy: 'WeightedRoundRobin'
|
133
|
+
}
|
134
|
+
}
|
135
|
+
end
|
136
|
+
|
137
|
+
specify do
|
138
|
+
expect(Shoryuken.polling_strategy('default')).to eq Shoryuken::Polling::WeightedRoundRobin
|
139
|
+
expect(Shoryuken.polling_strategy('group1')).to eq Shoryuken::Polling::WeightedRoundRobin
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
context 'when set to non-existent string' do
|
144
|
+
before do
|
145
|
+
Shoryuken.options[:polling_strategy] = 'NonExistent1'
|
146
|
+
|
147
|
+
Shoryuken.options[:groups] = {
|
148
|
+
'group1' => {
|
149
|
+
polling_strategy: 'NonExistent2'
|
150
|
+
}
|
151
|
+
}
|
152
|
+
end
|
153
|
+
|
154
|
+
specify do
|
155
|
+
expect { Shoryuken.polling_strategy('default') }.to raise_error(ArgumentError)
|
156
|
+
expect { Shoryuken.polling_strategy('group1') }.to raise_error(ArgumentError)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
context 'when set to a class' do
|
161
|
+
before do
|
162
|
+
class Foo < Shoryuken::Polling::BaseStrategy; end
|
163
|
+
class Bar < Shoryuken::Polling::BaseStrategy; end
|
164
|
+
|
165
|
+
Shoryuken.options[:polling_strategy] = Foo
|
166
|
+
|
167
|
+
Shoryuken.options[:groups] = {
|
168
|
+
'group1' => {
|
169
|
+
polling_strategy: Bar
|
170
|
+
}
|
171
|
+
}
|
172
|
+
end
|
173
|
+
|
174
|
+
specify do
|
175
|
+
expect(Shoryuken.polling_strategy('default')).to eq Foo
|
176
|
+
expect(Shoryuken.polling_strategy('group1')).to eq Bar
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
100
180
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -59,6 +59,8 @@ RSpec.configure do |config|
|
|
59
59
|
|
60
60
|
Aws.config[:stub_responses] = true
|
61
61
|
|
62
|
+
Shoryuken.sqs_client_receive_message_opts.clear
|
63
|
+
|
62
64
|
allow(Concurrent).to receive(:global_io_executor).and_return(Concurrent::ImmediateExecutor.new)
|
63
65
|
allow(Shoryuken).to receive(:active_job?).and_return(false)
|
64
66
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shoryuken
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.1.
|
4
|
+
version: 3.1.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pablo Cantero
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-07-
|
11
|
+
date: 2017-07-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|