shoryuken 3.0.11 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +1 -0
- data/CHANGELOG.md +13 -2
- data/Gemfile +1 -0
- data/README.md +15 -110
- data/bin/cli/sqs.rb +7 -0
- data/lib/shoryuken.rb +39 -173
- data/lib/shoryuken/default_worker_registry.rb +2 -2
- data/lib/shoryuken/environment_loader.rb +25 -9
- data/lib/shoryuken/fetcher.rb +17 -16
- data/lib/shoryuken/launcher.rb +86 -7
- data/lib/shoryuken/manager.rb +38 -79
- data/lib/shoryuken/options.rb +192 -0
- data/lib/shoryuken/polling/base.rb +67 -0
- data/lib/shoryuken/polling/strict_priority.rb +77 -0
- data/lib/shoryuken/polling/weighted_round_robin.rb +66 -0
- data/lib/shoryuken/processor.rb +21 -18
- data/lib/shoryuken/runner.rb +3 -15
- data/lib/shoryuken/version.rb +1 -1
- data/spec/integration/launcher_spec.rb +12 -6
- data/spec/shoryuken/environment_loader_spec.rb +3 -12
- data/spec/shoryuken/fetcher_spec.rb +30 -15
- data/spec/shoryuken/manager_spec.rb +9 -17
- data/spec/shoryuken/options_spec.rb +100 -0
- data/spec/shoryuken/{polling_spec.rb → polling/strict_priority_spec.rb} +1 -100
- data/spec/shoryuken/polling/weighted_round_robin_spec.rb +99 -0
- data/spec/shoryuken/processor_spec.rb +20 -39
- data/spec/shoryuken/runner_spec.rb +3 -4
- data/spec/shoryuken_spec.rb +0 -59
- data/spec/spec_helper.rb +7 -2
- metadata +12 -5
- data/lib/shoryuken/polling.rb +0 -204
@@ -0,0 +1,67 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
module Polling
|
3
|
+
QueueConfiguration = Struct.new(:name, :options) do
|
4
|
+
def hash
|
5
|
+
name.hash
|
6
|
+
end
|
7
|
+
|
8
|
+
def ==(other)
|
9
|
+
case other
|
10
|
+
when String
|
11
|
+
if options.empty?
|
12
|
+
name == other
|
13
|
+
else
|
14
|
+
false
|
15
|
+
end
|
16
|
+
else
|
17
|
+
super
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
alias_method :eql?, :==
|
22
|
+
|
23
|
+
def to_s
|
24
|
+
if options.empty?
|
25
|
+
name
|
26
|
+
else
|
27
|
+
"#<QueueConfiguration #{name} options=#{options.inspect}>"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
class BaseStrategy
|
33
|
+
include Util
|
34
|
+
|
35
|
+
def next_queue
|
36
|
+
fail NotImplementedError
|
37
|
+
end
|
38
|
+
|
39
|
+
def messages_found(queue, messages_found)
|
40
|
+
fail NotImplementedError
|
41
|
+
end
|
42
|
+
|
43
|
+
def active_queues
|
44
|
+
fail NotImplementedError
|
45
|
+
end
|
46
|
+
|
47
|
+
def ==(other)
|
48
|
+
case other
|
49
|
+
when Array
|
50
|
+
@queues == other
|
51
|
+
else
|
52
|
+
if other.respond_to?(:active_queues)
|
53
|
+
active_queues == other.active_queues
|
54
|
+
else
|
55
|
+
false
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
def delay
|
63
|
+
Shoryuken.options[:delay].to_f
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
module Polling
|
3
|
+
class StrictPriority < BaseStrategy
|
4
|
+
def initialize(queues)
|
5
|
+
# Priority ordering of the queues, highest priority first
|
6
|
+
@queues = queues
|
7
|
+
.group_by { |q| q }
|
8
|
+
.sort_by { |_, qs| -qs.count }
|
9
|
+
.map(&:first)
|
10
|
+
|
11
|
+
# Pause status of the queues, default to past time (unpaused)
|
12
|
+
@paused_until = queues
|
13
|
+
.each_with_object(Hash.new) { |queue, h| h[queue] = Time.at(0) }
|
14
|
+
|
15
|
+
# Start queues at 0
|
16
|
+
reset_next_queue
|
17
|
+
end
|
18
|
+
|
19
|
+
def next_queue
|
20
|
+
next_queue = next_active_queue
|
21
|
+
next_queue.nil? ? nil : QueueConfiguration.new(next_queue, {})
|
22
|
+
end
|
23
|
+
|
24
|
+
def messages_found(queue, messages_found)
|
25
|
+
if messages_found == 0
|
26
|
+
pause(queue)
|
27
|
+
else
|
28
|
+
reset_next_queue
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def active_queues
|
33
|
+
@queues
|
34
|
+
.reverse
|
35
|
+
.map.with_index(1)
|
36
|
+
.reject { |q, _| queue_paused?(q) }
|
37
|
+
.reverse
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def next_active_queue
|
43
|
+
reset_next_queue if queues_unpaused_since?
|
44
|
+
|
45
|
+
size = @queues.length
|
46
|
+
size.times do
|
47
|
+
queue = @queues[@next_queue_index]
|
48
|
+
@next_queue_index = (@next_queue_index + 1) % size
|
49
|
+
return queue unless queue_paused?(queue)
|
50
|
+
end
|
51
|
+
|
52
|
+
nil
|
53
|
+
end
|
54
|
+
|
55
|
+
def queues_unpaused_since?
|
56
|
+
last = @last_unpause_check
|
57
|
+
now = @last_unpause_check = Time.now
|
58
|
+
|
59
|
+
last && @paused_until.values.any? { |t| t > last && t <= now }
|
60
|
+
end
|
61
|
+
|
62
|
+
def reset_next_queue
|
63
|
+
@next_queue_index = 0
|
64
|
+
end
|
65
|
+
|
66
|
+
def queue_paused?(queue)
|
67
|
+
@paused_until[queue] > Time.now
|
68
|
+
end
|
69
|
+
|
70
|
+
def pause(queue)
|
71
|
+
return unless delay > 0
|
72
|
+
@paused_until[queue] = Time.now + delay
|
73
|
+
logger.debug "Paused #{queue}"
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
module Polling
|
3
|
+
class WeightedRoundRobin < BaseStrategy
|
4
|
+
def initialize(queues)
|
5
|
+
@initial_queues = queues
|
6
|
+
@queues = queues.dup.uniq
|
7
|
+
@paused_queues = []
|
8
|
+
end
|
9
|
+
|
10
|
+
def next_queue
|
11
|
+
unpause_queues
|
12
|
+
queue = @queues.shift
|
13
|
+
return nil if queue.nil?
|
14
|
+
|
15
|
+
@queues << queue
|
16
|
+
QueueConfiguration.new(queue, {})
|
17
|
+
end
|
18
|
+
|
19
|
+
def messages_found(queue, messages_found)
|
20
|
+
if messages_found == 0
|
21
|
+
pause(queue)
|
22
|
+
return
|
23
|
+
end
|
24
|
+
|
25
|
+
maximum_weight = maximum_queue_weight(queue)
|
26
|
+
current_weight = current_queue_weight(queue)
|
27
|
+
if maximum_weight > current_weight
|
28
|
+
logger.info { "Increasing #{queue} weight to #{current_weight + 1}, max: #{maximum_weight}" }
|
29
|
+
@queues << queue
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def active_queues
|
34
|
+
unparse_queues(@queues)
|
35
|
+
end
|
36
|
+
|
37
|
+
private
|
38
|
+
|
39
|
+
def pause(queue)
|
40
|
+
return unless @queues.delete(queue)
|
41
|
+
@paused_queues << [Time.now + delay, queue]
|
42
|
+
logger.debug "Paused #{queue}"
|
43
|
+
end
|
44
|
+
|
45
|
+
def unpause_queues
|
46
|
+
return if @paused_queues.empty?
|
47
|
+
return if Time.now < @paused_queues.first[0]
|
48
|
+
pause = @paused_queues.shift
|
49
|
+
@queues << pause[1]
|
50
|
+
logger.debug "Unpaused #{pause[1]}"
|
51
|
+
end
|
52
|
+
|
53
|
+
def current_queue_weight(queue)
|
54
|
+
queue_weight(@queues, queue)
|
55
|
+
end
|
56
|
+
|
57
|
+
def maximum_queue_weight(queue)
|
58
|
+
queue_weight(@initial_queues, queue)
|
59
|
+
end
|
60
|
+
|
61
|
+
def queue_weight(queues, queue)
|
62
|
+
queues.count { |q| q == queue }
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
data/lib/shoryuken/processor.rb
CHANGED
@@ -2,35 +2,41 @@ module Shoryuken
|
|
2
2
|
class Processor
|
3
3
|
include Util
|
4
4
|
|
5
|
-
|
6
|
-
|
5
|
+
attr_reader :queue, :sqs_msg
|
6
|
+
|
7
|
+
def initialize(queue, sqs_msg)
|
8
|
+
@queue = queue
|
9
|
+
@sqs_msg = sqs_msg
|
7
10
|
end
|
8
11
|
|
9
|
-
def process
|
10
|
-
worker
|
11
|
-
body = get_body(worker.class, sqs_msg)
|
12
|
+
def process
|
13
|
+
return logger.error { "No worker found for #{queue}" } unless worker
|
12
14
|
|
13
15
|
worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
|
14
16
|
worker.perform(sqs_msg, body)
|
15
17
|
end
|
16
18
|
rescue Exception => ex
|
17
|
-
|
19
|
+
logger.error { "Processor failed: #{ex.message}" }
|
20
|
+
logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
|
21
|
+
|
18
22
|
raise
|
19
|
-
ensure
|
20
|
-
@manager.processor_done(queue)
|
21
23
|
end
|
22
24
|
|
23
25
|
private
|
24
26
|
|
25
|
-
def
|
26
|
-
|
27
|
-
sqs_msg.map { |m| parse_body(worker_class, m) }
|
28
|
-
else
|
29
|
-
parse_body(worker_class, sqs_msg)
|
30
|
-
end
|
27
|
+
def worker
|
28
|
+
@_worker ||= Shoryuken.worker_registry.fetch_worker(queue, sqs_msg)
|
31
29
|
end
|
32
30
|
|
33
|
-
def
|
31
|
+
def worker_class
|
32
|
+
worker.class
|
33
|
+
end
|
34
|
+
|
35
|
+
def body
|
36
|
+
@_body ||= sqs_msg.is_a?(Array) ? sqs_msg.map(&method(:parse_body)) : parse_body(sqs_msg)
|
37
|
+
end
|
38
|
+
|
39
|
+
def parse_body(sqs_msg)
|
34
40
|
body_parser = worker_class.get_shoryuken_options['body_parser']
|
35
41
|
|
36
42
|
case body_parser
|
@@ -50,9 +56,6 @@ module Shoryuken
|
|
50
56
|
body_parser.load(sqs_msg.body)
|
51
57
|
end
|
52
58
|
end
|
53
|
-
rescue => ex
|
54
|
-
logger.error { "Error parsing the message body: #{ex.message}\nbody_parser: #{body_parser}\nsqs_msg.body: #{sqs_msg.body}" }
|
55
|
-
raise
|
56
59
|
end
|
57
60
|
end
|
58
61
|
end
|
data/lib/shoryuken/runner.rb
CHANGED
@@ -19,7 +19,7 @@ module Shoryuken
|
|
19
19
|
def run(options)
|
20
20
|
self_read, self_write = IO.pipe
|
21
21
|
|
22
|
-
%w(INT TERM USR1
|
22
|
+
%w(INT TERM USR1 TTIN).each do |sig|
|
23
23
|
begin
|
24
24
|
trap sig do
|
25
25
|
self_write.puts(sig)
|
@@ -43,22 +43,15 @@ module Shoryuken
|
|
43
43
|
|
44
44
|
@launcher = Shoryuken::Launcher.new
|
45
45
|
|
46
|
-
if (callback = Shoryuken.start_callback)
|
47
|
-
logger.info { 'Calling on_start callback' }
|
48
|
-
callback.call
|
49
|
-
end
|
50
|
-
|
51
|
-
fire_event(:startup)
|
52
|
-
|
53
46
|
begin
|
54
|
-
@launcher.
|
47
|
+
@launcher.start
|
55
48
|
|
56
49
|
while (readable_io = IO.select([self_read]))
|
57
50
|
signal = readable_io.first[0].gets.strip
|
58
51
|
handle_signal(signal)
|
59
52
|
end
|
60
53
|
rescue Interrupt
|
61
|
-
@launcher.stop
|
54
|
+
@launcher.stop!
|
62
55
|
exit 0
|
63
56
|
end
|
64
57
|
end
|
@@ -110,7 +103,6 @@ module Shoryuken
|
|
110
103
|
logger.info { 'Received USR1, will soft shutdown down' }
|
111
104
|
|
112
105
|
@launcher.stop
|
113
|
-
fire_event(:quiet, true)
|
114
106
|
exit 0
|
115
107
|
end
|
116
108
|
|
@@ -126,13 +118,9 @@ module Shoryuken
|
|
126
118
|
end
|
127
119
|
|
128
120
|
def handle_signal(sig)
|
129
|
-
logger.info { "Got #{sig} signal" }
|
130
|
-
|
131
121
|
case sig
|
132
122
|
when 'USR1' then execute_soft_shutdown
|
133
123
|
when 'TTIN' then print_threads_backtrace
|
134
|
-
when 'USR2'
|
135
|
-
logger.warn { "Received #{sig}, will do nothing. To execute soft shutdown, please send USR1" }
|
136
124
|
else
|
137
125
|
logger.info { "Received #{sig}, will shutdown down" }
|
138
126
|
|
data/lib/shoryuken/version.rb
CHANGED
@@ -6,25 +6,31 @@ require 'securerandom'
|
|
6
6
|
RSpec.describe Shoryuken::Launcher do
|
7
7
|
describe 'Consuming messages', slow: :true do
|
8
8
|
before do
|
9
|
+
Aws.config[:stub_responses] = false
|
10
|
+
Aws.config[:region] = 'us-east-1'
|
11
|
+
|
9
12
|
StandardWorker.received_messages = 0
|
10
13
|
|
11
|
-
queue = "
|
14
|
+
queue = "shoryuken-travis-#{StandardWorker}-#{SecureRandom.uuid}"
|
12
15
|
|
13
|
-
Shoryuken::Client.sqs.create_queue
|
16
|
+
Shoryuken::Client.sqs.create_queue(queue_name: queue)
|
14
17
|
|
15
|
-
Shoryuken.
|
18
|
+
Shoryuken.add_group('default', 1)
|
19
|
+
Shoryuken.add_queue(queue, 1, 'default')
|
16
20
|
|
17
21
|
StandardWorker.get_shoryuken_options['queue'] = queue
|
18
22
|
|
19
|
-
Shoryuken.register_worker
|
23
|
+
Shoryuken.register_worker(queue, StandardWorker)
|
20
24
|
end
|
21
25
|
|
22
26
|
after do
|
27
|
+
Aws.config[:stub_responses] = true
|
28
|
+
|
23
29
|
queue_url = Shoryuken::Client.sqs.get_queue_url(
|
24
30
|
queue_name: StandardWorker.get_shoryuken_options['queue']
|
25
31
|
).queue_url
|
26
32
|
|
27
|
-
Shoryuken::Client.sqs.delete_queue
|
33
|
+
Shoryuken::Client.sqs.delete_queue(queue_url: queue_url)
|
28
34
|
end
|
29
35
|
|
30
36
|
it 'consumes as a command worker' do
|
@@ -61,7 +67,7 @@ RSpec.describe Shoryuken::Launcher do
|
|
61
67
|
end
|
62
68
|
|
63
69
|
def poll_queues_until
|
64
|
-
subject.
|
70
|
+
subject.start
|
65
71
|
|
66
72
|
Timeout::timeout(10) do
|
67
73
|
begin
|
@@ -15,20 +15,11 @@ RSpec.describe Shoryuken::EnvironmentLoader do
|
|
15
15
|
allow(subject).to receive(:patch_deprecated_workers)
|
16
16
|
end
|
17
17
|
|
18
|
-
|
19
|
-
Shoryuken.options[:queues] = ['
|
18
|
+
specify do
|
19
|
+
Shoryuken.options[:queues] = ['queue1', ['queue2', 2]]
|
20
20
|
subject.load
|
21
21
|
|
22
|
-
expect(Shoryuken.queues).to eq(%w(
|
23
|
-
end
|
24
|
-
|
25
|
-
context 'with priority' do
|
26
|
-
it 'parses' do
|
27
|
-
Shoryuken.options[:queues] = ['queue_1', ['queue_2', 2]]
|
28
|
-
subject.load
|
29
|
-
|
30
|
-
expect(Shoryuken.queues).to eq(%w(queue_1 queue_2 queue_2))
|
31
|
-
end
|
22
|
+
expect(Shoryuken.groups['default'][:queues]).to eq(%w(queue1 queue2 queue2))
|
32
23
|
end
|
33
24
|
end
|
34
25
|
end
|
@@ -2,36 +2,51 @@ require 'spec_helper'
|
|
2
2
|
require 'shoryuken/manager'
|
3
3
|
require 'shoryuken/fetcher'
|
4
4
|
|
5
|
-
describe Shoryuken::Fetcher do
|
6
|
-
let(:queue)
|
7
|
-
let(:queue_name)
|
5
|
+
RSpec.describe Shoryuken::Fetcher do
|
6
|
+
let(:queue) { instance_double('Shoryuken::Queue') }
|
7
|
+
let(:queue_name) { 'default' }
|
8
8
|
let(:queue_config) { Shoryuken::Polling::QueueConfiguration.new(queue_name, {}) }
|
9
|
+
let(:group) { 'default' }
|
9
10
|
|
10
11
|
let(:sqs_msg) do
|
11
|
-
double(
|
12
|
+
double(
|
13
|
+
Shoryuken::Message,
|
12
14
|
queue_url: queue_name,
|
13
15
|
body: 'test',
|
14
16
|
message_id: 'fc754df79cc24c4196ca5996a44b771e',
|
15
|
-
|
17
|
+
)
|
16
18
|
end
|
17
19
|
|
18
|
-
subject { described_class.new }
|
20
|
+
subject { described_class.new(group) }
|
19
21
|
|
20
22
|
describe '#fetch' do
|
21
|
-
|
23
|
+
let(:limit) { 1 }
|
24
|
+
|
25
|
+
specify do
|
22
26
|
expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
27
|
+
|
28
|
+
Shoryuken.sqs_client_receive_message_opts[group] = { wait_time_seconds: 10 }
|
29
|
+
|
23
30
|
expect(queue).to receive(:receive_messages).
|
24
|
-
with(max_number_of_messages:
|
31
|
+
with(wait_time_seconds: 10, max_number_of_messages: limit, message_attribute_names: ['All'], attribute_names: ['All']).
|
25
32
|
and_return([])
|
26
|
-
|
33
|
+
|
34
|
+
subject.fetch(queue_config, limit)
|
27
35
|
end
|
28
36
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
37
|
+
context 'when limit is greater than FETCH_LIMIT' do
|
38
|
+
let(:limit) { 20 }
|
39
|
+
|
40
|
+
specify do
|
41
|
+
Shoryuken.sqs_client_receive_message_opts[group] = {}
|
42
|
+
|
43
|
+
allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
44
|
+
expect(queue).to receive(:receive_messages).
|
45
|
+
with(max_number_of_messages: described_class::FETCH_LIMIT, attribute_names: ['All'], message_attribute_names: ['All']).
|
46
|
+
and_return([])
|
47
|
+
|
48
|
+
subject.fetch(queue_config, limit)
|
49
|
+
end
|
35
50
|
end
|
36
51
|
end
|
37
52
|
end
|