shoryuken 3.0.11 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,67 @@
1
+ module Shoryuken
2
+ module Polling
3
+ QueueConfiguration = Struct.new(:name, :options) do
4
+ def hash
5
+ name.hash
6
+ end
7
+
8
+ def ==(other)
9
+ case other
10
+ when String
11
+ if options.empty?
12
+ name == other
13
+ else
14
+ false
15
+ end
16
+ else
17
+ super
18
+ end
19
+ end
20
+
21
+ alias_method :eql?, :==
22
+
23
+ def to_s
24
+ if options.empty?
25
+ name
26
+ else
27
+ "#<QueueConfiguration #{name} options=#{options.inspect}>"
28
+ end
29
+ end
30
+ end
31
+
32
+ class BaseStrategy
33
+ include Util
34
+
35
+ def next_queue
36
+ fail NotImplementedError
37
+ end
38
+
39
+ def messages_found(queue, messages_found)
40
+ fail NotImplementedError
41
+ end
42
+
43
+ def active_queues
44
+ fail NotImplementedError
45
+ end
46
+
47
+ def ==(other)
48
+ case other
49
+ when Array
50
+ @queues == other
51
+ else
52
+ if other.respond_to?(:active_queues)
53
+ active_queues == other.active_queues
54
+ else
55
+ false
56
+ end
57
+ end
58
+ end
59
+
60
+ private
61
+
62
+ def delay
63
+ Shoryuken.options[:delay].to_f
64
+ end
65
+ end
66
+ end
67
+ end
@@ -0,0 +1,77 @@
1
+ module Shoryuken
2
+ module Polling
3
+ class StrictPriority < BaseStrategy
4
+ def initialize(queues)
5
+ # Priority ordering of the queues, highest priority first
6
+ @queues = queues
7
+ .group_by { |q| q }
8
+ .sort_by { |_, qs| -qs.count }
9
+ .map(&:first)
10
+
11
+ # Pause status of the queues, default to past time (unpaused)
12
+ @paused_until = queues
13
+ .each_with_object(Hash.new) { |queue, h| h[queue] = Time.at(0) }
14
+
15
+ # Start queues at 0
16
+ reset_next_queue
17
+ end
18
+
19
+ def next_queue
20
+ next_queue = next_active_queue
21
+ next_queue.nil? ? nil : QueueConfiguration.new(next_queue, {})
22
+ end
23
+
24
+ def messages_found(queue, messages_found)
25
+ if messages_found == 0
26
+ pause(queue)
27
+ else
28
+ reset_next_queue
29
+ end
30
+ end
31
+
32
+ def active_queues
33
+ @queues
34
+ .reverse
35
+ .map.with_index(1)
36
+ .reject { |q, _| queue_paused?(q) }
37
+ .reverse
38
+ end
39
+
40
+ private
41
+
42
+ def next_active_queue
43
+ reset_next_queue if queues_unpaused_since?
44
+
45
+ size = @queues.length
46
+ size.times do
47
+ queue = @queues[@next_queue_index]
48
+ @next_queue_index = (@next_queue_index + 1) % size
49
+ return queue unless queue_paused?(queue)
50
+ end
51
+
52
+ nil
53
+ end
54
+
55
+ def queues_unpaused_since?
56
+ last = @last_unpause_check
57
+ now = @last_unpause_check = Time.now
58
+
59
+ last && @paused_until.values.any? { |t| t > last && t <= now }
60
+ end
61
+
62
+ def reset_next_queue
63
+ @next_queue_index = 0
64
+ end
65
+
66
+ def queue_paused?(queue)
67
+ @paused_until[queue] > Time.now
68
+ end
69
+
70
+ def pause(queue)
71
+ return unless delay > 0
72
+ @paused_until[queue] = Time.now + delay
73
+ logger.debug "Paused #{queue}"
74
+ end
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,66 @@
1
+ module Shoryuken
2
+ module Polling
3
+ class WeightedRoundRobin < BaseStrategy
4
+ def initialize(queues)
5
+ @initial_queues = queues
6
+ @queues = queues.dup.uniq
7
+ @paused_queues = []
8
+ end
9
+
10
+ def next_queue
11
+ unpause_queues
12
+ queue = @queues.shift
13
+ return nil if queue.nil?
14
+
15
+ @queues << queue
16
+ QueueConfiguration.new(queue, {})
17
+ end
18
+
19
+ def messages_found(queue, messages_found)
20
+ if messages_found == 0
21
+ pause(queue)
22
+ return
23
+ end
24
+
25
+ maximum_weight = maximum_queue_weight(queue)
26
+ current_weight = current_queue_weight(queue)
27
+ if maximum_weight > current_weight
28
+ logger.info { "Increasing #{queue} weight to #{current_weight + 1}, max: #{maximum_weight}" }
29
+ @queues << queue
30
+ end
31
+ end
32
+
33
+ def active_queues
34
+ unparse_queues(@queues)
35
+ end
36
+
37
+ private
38
+
39
+ def pause(queue)
40
+ return unless @queues.delete(queue)
41
+ @paused_queues << [Time.now + delay, queue]
42
+ logger.debug "Paused #{queue}"
43
+ end
44
+
45
+ def unpause_queues
46
+ return if @paused_queues.empty?
47
+ return if Time.now < @paused_queues.first[0]
48
+ pause = @paused_queues.shift
49
+ @queues << pause[1]
50
+ logger.debug "Unpaused #{pause[1]}"
51
+ end
52
+
53
+ def current_queue_weight(queue)
54
+ queue_weight(@queues, queue)
55
+ end
56
+
57
+ def maximum_queue_weight(queue)
58
+ queue_weight(@initial_queues, queue)
59
+ end
60
+
61
+ def queue_weight(queues, queue)
62
+ queues.count { |q| q == queue }
63
+ end
64
+ end
65
+ end
66
+ end
@@ -2,35 +2,41 @@ module Shoryuken
2
2
  class Processor
3
3
  include Util
4
4
 
5
- def initialize(manager)
6
- @manager = manager
5
+ attr_reader :queue, :sqs_msg
6
+
7
+ def initialize(queue, sqs_msg)
8
+ @queue = queue
9
+ @sqs_msg = sqs_msg
7
10
  end
8
11
 
9
- def process(queue, sqs_msg)
10
- worker = Shoryuken.worker_registry.fetch_worker(queue, sqs_msg)
11
- body = get_body(worker.class, sqs_msg)
12
+ def process
13
+ return logger.error { "No worker found for #{queue}" } unless worker
12
14
 
13
15
  worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
14
16
  worker.perform(sqs_msg, body)
15
17
  end
16
18
  rescue Exception => ex
17
- @manager.processor_failed(ex)
19
+ logger.error { "Processor failed: #{ex.message}" }
20
+ logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
21
+
18
22
  raise
19
- ensure
20
- @manager.processor_done(queue)
21
23
  end
22
24
 
23
25
  private
24
26
 
25
- def get_body(worker_class, sqs_msg)
26
- if sqs_msg.is_a? Array
27
- sqs_msg.map { |m| parse_body(worker_class, m) }
28
- else
29
- parse_body(worker_class, sqs_msg)
30
- end
27
+ def worker
28
+ @_worker ||= Shoryuken.worker_registry.fetch_worker(queue, sqs_msg)
31
29
  end
32
30
 
33
- def parse_body(worker_class, sqs_msg)
31
+ def worker_class
32
+ worker.class
33
+ end
34
+
35
+ def body
36
+ @_body ||= sqs_msg.is_a?(Array) ? sqs_msg.map(&method(:parse_body)) : parse_body(sqs_msg)
37
+ end
38
+
39
+ def parse_body(sqs_msg)
34
40
  body_parser = worker_class.get_shoryuken_options['body_parser']
35
41
 
36
42
  case body_parser
@@ -50,9 +56,6 @@ module Shoryuken
50
56
  body_parser.load(sqs_msg.body)
51
57
  end
52
58
  end
53
- rescue => ex
54
- logger.error { "Error parsing the message body: #{ex.message}\nbody_parser: #{body_parser}\nsqs_msg.body: #{sqs_msg.body}" }
55
- raise
56
59
  end
57
60
  end
58
61
  end
@@ -19,7 +19,7 @@ module Shoryuken
19
19
  def run(options)
20
20
  self_read, self_write = IO.pipe
21
21
 
22
- %w(INT TERM USR1 USR2 TTIN).each do |sig|
22
+ %w(INT TERM USR1 TTIN).each do |sig|
23
23
  begin
24
24
  trap sig do
25
25
  self_write.puts(sig)
@@ -43,22 +43,15 @@ module Shoryuken
43
43
 
44
44
  @launcher = Shoryuken::Launcher.new
45
45
 
46
- if (callback = Shoryuken.start_callback)
47
- logger.info { 'Calling on_start callback' }
48
- callback.call
49
- end
50
-
51
- fire_event(:startup)
52
-
53
46
  begin
54
- @launcher.run
47
+ @launcher.start
55
48
 
56
49
  while (readable_io = IO.select([self_read]))
57
50
  signal = readable_io.first[0].gets.strip
58
51
  handle_signal(signal)
59
52
  end
60
53
  rescue Interrupt
61
- @launcher.stop(shutdown: true)
54
+ @launcher.stop!
62
55
  exit 0
63
56
  end
64
57
  end
@@ -110,7 +103,6 @@ module Shoryuken
110
103
  logger.info { 'Received USR1, will soft shutdown down' }
111
104
 
112
105
  @launcher.stop
113
- fire_event(:quiet, true)
114
106
  exit 0
115
107
  end
116
108
 
@@ -126,13 +118,9 @@ module Shoryuken
126
118
  end
127
119
 
128
120
  def handle_signal(sig)
129
- logger.info { "Got #{sig} signal" }
130
-
131
121
  case sig
132
122
  when 'USR1' then execute_soft_shutdown
133
123
  when 'TTIN' then print_threads_backtrace
134
- when 'USR2'
135
- logger.warn { "Received #{sig}, will do nothing. To execute soft shutdown, please send USR1" }
136
124
  else
137
125
  logger.info { "Received #{sig}, will shutdown down" }
138
126
 
@@ -1,3 +1,3 @@
1
1
  module Shoryuken
2
- VERSION = '3.0.11'.freeze
2
+ VERSION = '3.1.0'.freeze
3
3
  end
@@ -6,25 +6,31 @@ require 'securerandom'
6
6
  RSpec.describe Shoryuken::Launcher do
7
7
  describe 'Consuming messages', slow: :true do
8
8
  before do
9
+ Aws.config[:stub_responses] = false
10
+ Aws.config[:region] = 'us-east-1'
11
+
9
12
  StandardWorker.received_messages = 0
10
13
 
11
- queue = "test_shoryuken#{StandardWorker}_#{SecureRandom.uuid}"
14
+ queue = "shoryuken-travis-#{StandardWorker}-#{SecureRandom.uuid}"
12
15
 
13
- Shoryuken::Client.sqs.create_queue queue_name: queue
16
+ Shoryuken::Client.sqs.create_queue(queue_name: queue)
14
17
 
15
- Shoryuken.queues << queue
18
+ Shoryuken.add_group('default', 1)
19
+ Shoryuken.add_queue(queue, 1, 'default')
16
20
 
17
21
  StandardWorker.get_shoryuken_options['queue'] = queue
18
22
 
19
- Shoryuken.register_worker queue, StandardWorker
23
+ Shoryuken.register_worker(queue, StandardWorker)
20
24
  end
21
25
 
22
26
  after do
27
+ Aws.config[:stub_responses] = true
28
+
23
29
  queue_url = Shoryuken::Client.sqs.get_queue_url(
24
30
  queue_name: StandardWorker.get_shoryuken_options['queue']
25
31
  ).queue_url
26
32
 
27
- Shoryuken::Client.sqs.delete_queue queue_url: queue_url
33
+ Shoryuken::Client.sqs.delete_queue(queue_url: queue_url)
28
34
  end
29
35
 
30
36
  it 'consumes as a command worker' do
@@ -61,7 +67,7 @@ RSpec.describe Shoryuken::Launcher do
61
67
  end
62
68
 
63
69
  def poll_queues_until
64
- subject.run
70
+ subject.start
65
71
 
66
72
  Timeout::timeout(10) do
67
73
  begin
@@ -15,20 +15,11 @@ RSpec.describe Shoryuken::EnvironmentLoader do
15
15
  allow(subject).to receive(:patch_deprecated_workers)
16
16
  end
17
17
 
18
- it 'parses' do
19
- Shoryuken.options[:queues] = ['queue_1']
18
+ specify do
19
+ Shoryuken.options[:queues] = ['queue1', ['queue2', 2]]
20
20
  subject.load
21
21
 
22
- expect(Shoryuken.queues).to eq(%w(queue_1))
23
- end
24
-
25
- context 'with priority' do
26
- it 'parses' do
27
- Shoryuken.options[:queues] = ['queue_1', ['queue_2', 2]]
28
- subject.load
29
-
30
- expect(Shoryuken.queues).to eq(%w(queue_1 queue_2 queue_2))
31
- end
22
+ expect(Shoryuken.groups['default'][:queues]).to eq(%w(queue1 queue2 queue2))
32
23
  end
33
24
  end
34
25
  end
@@ -2,36 +2,51 @@ require 'spec_helper'
2
2
  require 'shoryuken/manager'
3
3
  require 'shoryuken/fetcher'
4
4
 
5
- describe Shoryuken::Fetcher do
6
- let(:queue) { instance_double('Shoryuken::Queue') }
7
- let(:queue_name) { 'default' }
5
+ RSpec.describe Shoryuken::Fetcher do
6
+ let(:queue) { instance_double('Shoryuken::Queue') }
7
+ let(:queue_name) { 'default' }
8
8
  let(:queue_config) { Shoryuken::Polling::QueueConfiguration.new(queue_name, {}) }
9
+ let(:group) { 'default' }
9
10
 
10
11
  let(:sqs_msg) do
11
- double(Shoryuken::Message,
12
+ double(
13
+ Shoryuken::Message,
12
14
  queue_url: queue_name,
13
15
  body: 'test',
14
16
  message_id: 'fc754df79cc24c4196ca5996a44b771e',
15
- )
17
+ )
16
18
  end
17
19
 
18
- subject { described_class.new }
20
+ subject { described_class.new(group) }
19
21
 
20
22
  describe '#fetch' do
21
- it 'calls Shoryuken::Client to receive messages' do
23
+ let(:limit) { 1 }
24
+
25
+ specify do
22
26
  expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
27
+
28
+ Shoryuken.sqs_client_receive_message_opts[group] = { wait_time_seconds: 10 }
29
+
23
30
  expect(queue).to receive(:receive_messages).
24
- with(max_number_of_messages: 1, attribute_names: ['All'], message_attribute_names: ['All']).
31
+ with(wait_time_seconds: 10, max_number_of_messages: limit, message_attribute_names: ['All'], attribute_names: ['All']).
25
32
  and_return([])
26
- subject.fetch(queue_config, 1)
33
+
34
+ subject.fetch(queue_config, limit)
27
35
  end
28
36
 
29
- it 'maxes messages to receive to 10 (SQS limit)' do
30
- allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
31
- expect(queue).to receive(:receive_messages).
32
- with(max_number_of_messages: 10, attribute_names: ['All'], message_attribute_names: ['All']).
33
- and_return([])
34
- subject.fetch(queue_config, 20)
37
+ context 'when limit is greater than FETCH_LIMIT' do
38
+ let(:limit) { 20 }
39
+
40
+ specify do
41
+ Shoryuken.sqs_client_receive_message_opts[group] = {}
42
+
43
+ allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
44
+ expect(queue).to receive(:receive_messages).
45
+ with(max_number_of_messages: described_class::FETCH_LIMIT, attribute_names: ['All'], message_attribute_names: ['All']).
46
+ and_return([])
47
+
48
+ subject.fetch(queue_config, limit)
49
+ end
35
50
  end
36
51
  end
37
52
  end