shoryuken 2.1.1 → 2.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,206 @@
1
+ module Shoryuken
2
+ module Polling
3
+ QueueConfiguration = Struct.new(:name, :options) do
4
+ def hash
5
+ name.hash
6
+ end
7
+
8
+ def ==(other)
9
+ case other
10
+ when String
11
+ if options.empty?
12
+ name == other
13
+ else
14
+ false
15
+ end
16
+ else
17
+ super
18
+ end
19
+ end
20
+
21
+ alias_method :eql?, :==
22
+
23
+ def to_s
24
+ if options.empty?
25
+ name
26
+ else
27
+ "#<QueueConfiguration #{name} options=#{options.inspect}>"
28
+ end
29
+ end
30
+ end
31
+
32
+ class BaseStrategy
33
+ include Util
34
+
35
+ def next_queue
36
+ fail NotImplementedError
37
+ end
38
+
39
+ def messages_found(queue, messages_found)
40
+ fail NotImplementedError
41
+ end
42
+
43
+ def active_queues
44
+ fail NotImplementedError
45
+ end
46
+
47
+ def ==(other)
48
+ case other
49
+ when Array
50
+ @queues == other
51
+ else
52
+ if other.respond_to?(:active_queues)
53
+ active_queues == other.active_queues
54
+ else
55
+ false
56
+ end
57
+ end
58
+ end
59
+
60
+ private
61
+
62
+ def delay
63
+ Shoryuken.options[:delay].to_f
64
+ end
65
+ end
66
+
67
+ class WeightedRoundRobin < BaseStrategy
68
+
69
+ def initialize(queues)
70
+ @initial_queues = queues
71
+ @queues = queues.dup.uniq
72
+ @paused_queues = []
73
+ end
74
+
75
+ def next_queue
76
+ unpause_queues
77
+ queue = @queues.shift
78
+ return nil if queue.nil?
79
+
80
+ @queues << queue
81
+ QueueConfiguration.new(queue, {})
82
+ end
83
+
84
+ def messages_found(queue, messages_found)
85
+ if messages_found == 0
86
+ pause(queue)
87
+ return
88
+ end
89
+
90
+ maximum_weight = maximum_queue_weight(queue)
91
+ current_weight = current_queue_weight(queue)
92
+ if maximum_weight > current_weight
93
+ logger.info { "Increasing '#{queue}' weight to #{current_weight + 1}, max: #{maximum_weight}" }
94
+ @queues << queue
95
+ end
96
+ end
97
+
98
+ def active_queues
99
+ unparse_queues(@queues)
100
+ end
101
+
102
+ private
103
+
104
+ def pause(queue)
105
+ return unless @queues.delete(queue)
106
+ @paused_queues << [Time.now + delay, queue]
107
+ logger.debug "Paused '#{queue}'"
108
+ end
109
+
110
+ def unpause_queues
111
+ return if @paused_queues.empty?
112
+ return if Time.now < @paused_queues.first[0]
113
+ pause = @paused_queues.shift
114
+ @queues << pause[1]
115
+ logger.debug "Unpaused '#{pause[1]}'"
116
+ end
117
+
118
+ def current_queue_weight(queue)
119
+ queue_weight(@queues, queue)
120
+ end
121
+
122
+ def maximum_queue_weight(queue)
123
+ queue_weight(@initial_queues, queue)
124
+ end
125
+
126
+ def queue_weight(queues, queue)
127
+ queues.count { |q| q == queue }
128
+ end
129
+ end
130
+
131
+ class StrictPriority < BaseStrategy
132
+
133
+ def initialize(queues)
134
+ # Priority ordering of the queues, highest priority first
135
+ @queues = queues
136
+ .group_by { |q| q }
137
+ .sort_by { |_, qs| -qs.count }
138
+ .map(&:first)
139
+
140
+ # Pause status of the queues, default to past time (unpaused)
141
+ @paused_until = queues
142
+ .each_with_object(Hash.new) { |queue, h| h[queue] = Time.at(0) }
143
+
144
+ # Start queues at 0
145
+ reset_next_queue
146
+ end
147
+
148
+ def next_queue
149
+ next_queue = next_active_queue
150
+ next_queue.nil? ? nil : QueueConfiguration.new(next_queue, {})
151
+ end
152
+
153
+ def messages_found(queue, messages_found)
154
+ if messages_found == 0
155
+ pause(queue)
156
+ else
157
+ reset_next_queue
158
+ end
159
+ end
160
+
161
+ def active_queues
162
+ @queues
163
+ .reverse
164
+ .map.with_index(1)
165
+ .reject { |q, _| queue_paused?(q) }
166
+ .reverse
167
+ end
168
+
169
+ private
170
+
171
+ def next_active_queue
172
+ reset_next_queue if queues_unpaused_since?
173
+
174
+ size = @queues.length
175
+ size.times do
176
+ queue = @queues[@next_queue_index]
177
+ @next_queue_index = (@next_queue_index + 1) % size
178
+ return queue unless queue_paused?(queue)
179
+ end
180
+
181
+ return nil
182
+ end
183
+
184
+ def queues_unpaused_since?
185
+ last = @last_unpause_check
186
+ now = @last_unpause_check = Time.now
187
+
188
+ last && @paused_until.values.any? { |t| t > last && t <= now }
189
+ end
190
+
191
+ def reset_next_queue
192
+ @next_queue_index = 0
193
+ end
194
+
195
+ def queue_paused?(queue)
196
+ @paused_until[queue] > Time.now
197
+ end
198
+
199
+ def pause(queue)
200
+ return unless delay > 0
201
+ @paused_until[queue] = Time.now + delay
202
+ logger.debug "Paused '#{queue}'"
203
+ end
204
+ end
205
+ end
206
+ end
@@ -12,17 +12,18 @@ module Shoryuken
12
12
  attr_accessor :proxy_id
13
13
 
14
14
  def process(queue, sqs_msg)
15
- @manager.async.real_thread(proxy_id, Thread.current)
16
-
17
15
  worker = Shoryuken.worker_registry.fetch_worker(queue, sqs_msg)
16
+ body = get_body(worker.class, sqs_msg)
18
17
 
19
- body = get_body(worker.class, sqs_msg)
18
+ worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
19
+ worker.perform(sqs_msg, body)
20
+ end
20
21
 
21
- worker.class.server_middleware.invoke(worker, queue, sqs_msg, body) do
22
- worker.perform(sqs_msg, body)
23
- end
22
+ @manager.async.processor_done(queue, current_actor)
23
+ end
24
24
 
25
- @manager.async.processor_done(queue, current_actor)
25
+ def running_thread
26
+ Thread.current
26
27
  end
27
28
 
28
29
  private
@@ -1,3 +1,3 @@
1
1
  module Shoryuken
2
- VERSION = '2.1.1'
2
+ VERSION = '2.1.2'
3
3
  end
@@ -3,69 +3,35 @@ require 'shoryuken/manager'
3
3
  require 'shoryuken/fetcher'
4
4
 
5
5
  describe Shoryuken::Fetcher do
6
- let(:manager) { double Shoryuken::Manager }
7
- let(:queue) { double Shoryuken::Queue }
6
+ let(:queue) { instance_double('Shoryuken::Queue') }
8
7
  let(:queue_name) { 'default' }
8
+ let(:queue_config) { Shoryuken::Polling::QueueConfiguration.new(queue_name, {}) }
9
9
 
10
10
  let(:sqs_msg) do
11
- double Shoryuken::Message,
11
+ double(Shoryuken::Message,
12
12
  queue_url: queue_name,
13
13
  body: 'test',
14
- message_id: 'fc754df7-9cc2-4c41-96ca-5996a44b771e'
15
- end
16
-
17
- subject { described_class.new(manager) }
18
-
19
- before do
20
- allow(manager).to receive(:async).and_return(manager)
21
- allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
14
+ message_id: 'fc754df79cc24c4196ca5996a44b771e',
15
+ )
22
16
  end
23
17
 
18
+ subject { described_class.new }
24
19
 
25
20
  describe '#fetch' do
26
- it 'calls pause when no message' do
27
- allow(queue).to receive(:receive_messages).with(max_number_of_messages: 1, attribute_names: ['All'], message_attribute_names: ['All']).and_return([])
28
-
29
- expect(manager).to receive(:pause_queue!).with(queue_name)
30
- expect(manager).to receive(:dispatch)
31
-
32
- subject.fetch(queue_name, 1)
33
- end
34
-
35
- it 'assigns messages' do
36
- allow(queue).to receive(:receive_messages).with(max_number_of_messages: 5, attribute_names: ['All'], message_attribute_names: ['All']).and_return(sqs_msg)
37
-
38
- expect(manager).to receive(:rebalance_queue_weight!).with(queue_name)
39
- expect(manager).to receive(:assign).with(queue_name, sqs_msg)
40
- expect(manager).to receive(:dispatch)
41
-
42
- subject.fetch(queue_name, 5)
21
+ it 'calls Shoryuken::Client to receive messages' do
22
+ expect(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
23
+ expect(queue).to receive(:receive_messages).
24
+ with(max_number_of_messages: 1, attribute_names: ['All'], message_attribute_names: ['All']).
25
+ and_return([])
26
+ subject.fetch(queue_config, 1)
43
27
  end
44
28
 
45
- it 'assigns messages in batch' do
46
- TestWorker.get_shoryuken_options['batch'] = true
47
-
48
- allow(queue).to receive(:receive_messages).with(max_number_of_messages: described_class::FETCH_LIMIT, attribute_names: ['All'], message_attribute_names: ['All']).and_return(sqs_msg)
49
-
50
- expect(manager).to receive(:rebalance_queue_weight!).with(queue_name)
51
- expect(manager).to receive(:assign).with(queue_name, [sqs_msg])
52
- expect(manager).to receive(:dispatch)
53
-
54
- subject.fetch(queue_name, 5)
55
- end
56
-
57
- context 'when worker not found' do
58
- let(:queue_name) { 'notfound' }
59
-
60
- it 'ignores batch' do
61
- allow(queue).to receive(:receive_messages).with(max_number_of_messages: 5, attribute_names: ['All'], message_attribute_names: ['All']).and_return(sqs_msg)
62
-
63
- expect(manager).to receive(:rebalance_queue_weight!).with(queue_name)
64
- expect(manager).to receive(:assign).with(queue_name, sqs_msg)
65
- expect(manager).to receive(:dispatch)
66
-
67
- subject.fetch(queue_name, 5)
68
- end
29
+ it 'maxes messages to receive to 10 (SQS limit)' do
30
+ allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
31
+ expect(queue).to receive(:receive_messages).
32
+ with(max_number_of_messages: 10, attribute_names: ['All'], message_attribute_names: ['All']).
33
+ and_return([])
34
+ subject.fetch(queue_config, 20)
69
35
  end
70
36
  end
71
37
  end
@@ -1,11 +1,37 @@
1
1
  require 'spec_helper'
2
2
  require 'shoryuken/manager'
3
3
 
4
+ RSpec::Matchers.define :queue_config_of do |expected|
5
+ match do |actual|
6
+ actual.name == expected
7
+ end
8
+ end
9
+
4
10
  RSpec.describe Shoryuken::Manager do
5
- subject do
11
+ let(:queue) { 'default' }
12
+ let(:queues) { [queue] }
13
+ let(:polling_strategy) { Shoryuken::Polling::WeightedRoundRobin.new(queues) }
14
+ let(:fetcher) { Shoryuken::Fetcher.new }
15
+ let(:condvar) do
6
16
  condvar = double(:condvar)
7
17
  allow(condvar).to receive(:signal).and_return(nil)
8
- Shoryuken::Manager.new(condvar)
18
+ condvar
19
+ end
20
+ let(:async_manager) { instance_double(described_class.name) }
21
+ let(:concurrency) { 1 }
22
+
23
+ subject { Shoryuken::Manager.new(condvar) }
24
+
25
+ before(:each) do
26
+ Shoryuken.options[:concurrency] = concurrency
27
+ subject.fetcher = fetcher
28
+ subject.polling_strategy = polling_strategy
29
+ allow_any_instance_of(described_class).to receive(:async).and_return(async_manager)
30
+ end
31
+
32
+ after(:each) do
33
+ Shoryuken.options[:concurrency] = 1
34
+ TestWorker.get_shoryuken_options['batch'] = false
9
35
  end
10
36
 
11
37
  describe 'Invalid concurrency setting' do
@@ -14,109 +40,52 @@ RSpec.describe Shoryuken::Manager do
14
40
  expect { Shoryuken::Manager.new(nil) }
15
41
  .to raise_error(ArgumentError, 'Concurrency value -1 is invalid, it needs to be a positive number')
16
42
  end
17
-
18
43
  end
19
44
 
20
- describe 'Auto Scaling' do
21
- it 'decreases weight' do
22
- queue1 = 'shoryuken'
23
- queue2 = 'uppercut'
24
-
25
- Shoryuken.queues.clear
26
- # [shoryuken, 2]
27
- # [uppercut, 1]
28
- Shoryuken.queues << queue1
29
- Shoryuken.queues << queue1
30
- Shoryuken.queues << queue2
31
-
32
- expect(subject.instance_variable_get('@queues')).to eq [queue1, queue2]
33
-
34
- subject.pause_queue!(queue1)
35
-
36
- expect(subject.instance_variable_get('@queues')).to eq [queue2]
45
+ describe '#dispatch' do
46
+ it 'pauses when there are no active queues' do
47
+ expect(polling_strategy).to receive(:next_queue).and_return(nil)
48
+ expect_any_instance_of(described_class).to receive(:after)
49
+ subject.dispatch
37
50
  end
38
51
 
39
- it 'increases weight' do
40
- queue1 = 'shoryuken'
41
- queue2 = 'uppercut'
42
-
43
- Shoryuken.queues.clear
44
- # [shoryuken, 3]
45
- # [uppercut, 1]
46
- Shoryuken.queues << queue1
47
- Shoryuken.queues << queue1
48
- Shoryuken.queues << queue1
49
- Shoryuken.queues << queue2
50
-
51
- expect(subject.instance_variable_get('@queues')).to eq [queue1, queue2]
52
- subject.pause_queue!(queue1)
53
- expect(subject.instance_variable_get('@queues')).to eq [queue2]
54
-
55
- subject.rebalance_queue_weight!(queue1)
56
- expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1]
57
-
58
- subject.rebalance_queue_weight!(queue1)
59
- expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1, queue1]
60
-
61
- subject.rebalance_queue_weight!(queue1)
62
- expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1, queue1, queue1]
52
+ it 'calls dispatch_batch if worker wants batches' do
53
+ TestWorker.get_shoryuken_options['batch'] = true
54
+ expect_any_instance_of(described_class).to receive(:dispatch_batch).with(queue_config_of(queue))
55
+ expect_any_instance_of(described_class).to receive(:async).and_return(async_manager)
56
+ expect(async_manager).to receive(:dispatch)
57
+ subject.dispatch
63
58
  end
64
59
 
65
- it 'adds queue back' do
66
- queue1 = 'shoryuken'
67
- queue2 = 'uppercut'
68
-
69
- Shoryuken.queues.clear
70
- # [shoryuken, 2]
71
- # [uppercut, 1]
72
- Shoryuken.queues << queue1
73
- Shoryuken.queues << queue1
74
- Shoryuken.queues << queue2
75
-
76
- Shoryuken.options[:delay] = 0.1
77
-
78
- fetcher = double('Fetcher').as_null_object
79
- subject.fetcher = fetcher
80
-
81
- subject.pause_queue!(queue1)
82
- expect(subject.instance_variable_get('@queues')).to eq [queue2]
83
-
84
- sleep 0.5
85
-
86
- expect(subject.instance_variable_get('@queues')).to eq [queue2, queue1]
60
+ it 'calls dispatch_single_messages if worker wants single messages' do
61
+ expect_any_instance_of(described_class).to receive(:dispatch_single_messages).
62
+ with(queue_config_of(queue))
63
+ expect(async_manager).to receive(:dispatch)
64
+ subject.dispatch
87
65
  end
88
66
  end
89
67
 
90
- describe '#next_queue' do
91
- it 'returns queues' do
92
- queue1 = 'shoryuken'
93
- queue2 = 'uppercut'
94
-
95
- Shoryuken.queues.clear
96
-
97
- Shoryuken.register_worker queue1, TestWorker
98
- Shoryuken.register_worker queue2, TestWorker
99
-
100
- Shoryuken.queues << queue1
101
- Shoryuken.queues << queue2
102
-
103
- expect(subject.send :next_queue).to eq queue1
104
- expect(subject.send :next_queue).to eq queue2
68
+ describe '#dispatch_batch' do
69
+ it 'assings batch as a single message' do
70
+ q = polling_strategy.next_queue
71
+ messages = [1, 2, 3]
72
+ expect(fetcher).to receive(:fetch).with(q, described_class::BATCH_LIMIT).and_return(messages)
73
+ expect_any_instance_of(described_class).to receive(:assign).with(q.name, messages)
74
+ subject.send(:dispatch_batch, q)
105
75
  end
76
+ end
106
77
 
107
- it 'skips when no worker' do
108
- queue1 = 'shoryuken'
109
- queue2 = 'uppercut'
110
-
111
- Shoryuken.queues.clear
112
-
113
- Shoryuken.register_worker queue2, TestWorker
114
-
115
- Shoryuken.queues << queue1
116
- Shoryuken.queues << queue2
117
-
118
- expect(subject.send :next_queue).to eq queue2
119
- expect(subject.send :next_queue).to eq queue2
78
+ describe '#dispatch_single_messages' do
79
+ let(:concurrency) { 3 }
80
+
81
+ it 'assings messages from batch one by one' do
82
+ q = polling_strategy.next_queue
83
+ messages = [1, 2, 3]
84
+ expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
85
+ expect_any_instance_of(described_class).to receive(:assign).with(q.name, 1)
86
+ expect_any_instance_of(described_class).to receive(:assign).with(q.name, 2)
87
+ expect_any_instance_of(described_class).to receive(:assign).with(q.name, 3)
88
+ subject.send(:dispatch_single_messages, q)
120
89
  end
121
90
  end
122
91
  end