shoryuken 4.0.3 → 5.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -1
- data/.travis.yml +6 -5
- data/CHANGELOG.md +11 -0
- data/lib/shoryuken.rb +8 -3
- data/lib/shoryuken/environment_loader.rb +1 -1
- data/lib/shoryuken/fetcher.rb +19 -13
- data/lib/shoryuken/launcher.rb +1 -1
- data/lib/shoryuken/manager.rb +4 -4
- data/lib/shoryuken/options.rb +131 -193
- data/lib/shoryuken/queue.rb +1 -1
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker.rb +3 -3
- data/shoryuken.gemspec +0 -1
- data/spec/shoryuken/fetcher_spec.rb +17 -2
- data/spec/shoryuken/manager_spec.rb +35 -2
- data/spec/shoryuken/options_spec.rb +28 -26
- data/spec/shoryuken/processor_spec.rb +4 -4
- data/spec/shoryuken/queue_spec.rb +13 -0
- metadata +4 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ce36f4dac1c75e383185df15db196aec57c968dda1a5823b8d944ad4a9675906
|
4
|
+
data.tar.gz: a460a585e97a6c0a3432f6b28bec254d494900672a06bffbd7461e830a8de0cf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 833bab295345b25456f617f36e4ec2f2cfd0d823a01ddb87d8f528597ab79b91ce0ae8c67b47ffc2a9d1400f0a5cf62df3125fbdfbb49abd3a5e4a5eb3b7b0ee
|
7
|
+
data.tar.gz: 91a4355cd9e180f006e8768bde9627c1df9cf0057d294bdfcb422f479de2ef1784aa5b904c7b97dac8ed7bb7b9879bdea56eb4cd00d51ff456d3b4555f88c991
|
data/.rubocop.yml
CHANGED
data/.travis.yml
CHANGED
@@ -3,10 +3,11 @@ language: ruby
|
|
3
3
|
rvm:
|
4
4
|
- 2.0.0
|
5
5
|
- 2.1.10
|
6
|
-
- 2.2.
|
7
|
-
- 2.3.
|
8
|
-
- 2.4.
|
9
|
-
- 2.5.
|
6
|
+
- 2.2.10
|
7
|
+
- 2.3.8
|
8
|
+
- 2.4.4
|
9
|
+
- 2.5.1
|
10
|
+
- 2.6.3
|
10
11
|
|
11
12
|
notifications:
|
12
13
|
email:
|
@@ -23,7 +24,7 @@ env:
|
|
23
24
|
script: bundle exec rspec spec
|
24
25
|
|
25
26
|
before_install:
|
26
|
-
- gem
|
27
|
+
- gem install bundler -v '< 2'
|
27
28
|
|
28
29
|
after_success:
|
29
30
|
- bundle exec codeclimate-test-reporter
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,14 @@
|
|
1
|
+
## [v5.0.0] - 2019-06-18
|
2
|
+
|
3
|
+
- Fix bug where empty queues were not paused in batch processing mode
|
4
|
+
- [#569](https://github.com/phstc/shoryuken/pull/569)
|
5
|
+
|
6
|
+
- Preserve batch limit when receiving messages from a FIFO queue
|
7
|
+
- [#563](https://github.com/phstc/shoryuken/pull/563)
|
8
|
+
|
9
|
+
- Replace static options with instance options
|
10
|
+
- [#534](https://github.com/phstc/shoryuken/pull/534)
|
11
|
+
|
1
12
|
## [v4.0.3] - 2019-01-06
|
2
13
|
|
3
14
|
- Support delay per processing group
|
data/lib/shoryuken.rb
CHANGED
@@ -41,8 +41,12 @@ require 'shoryuken/options'
|
|
41
41
|
module Shoryuken
|
42
42
|
extend SingleForwardable
|
43
43
|
|
44
|
+
def self.shoryuken_options
|
45
|
+
@_shoryuken_options ||= Shoryuken::Options.new
|
46
|
+
end
|
47
|
+
|
44
48
|
def_delegators(
|
45
|
-
:
|
49
|
+
:shoryuken_options,
|
46
50
|
:active_job?,
|
47
51
|
:add_group,
|
48
52
|
:groups,
|
@@ -59,7 +63,7 @@ module Shoryuken
|
|
59
63
|
:start_callback=,
|
60
64
|
:stop_callback,
|
61
65
|
:stop_callback=,
|
62
|
-
:active_job_queue_name_prefixing
|
66
|
+
:active_job_queue_name_prefixing?,
|
63
67
|
:active_job_queue_name_prefixing=,
|
64
68
|
:sqs_client,
|
65
69
|
:sqs_client=,
|
@@ -79,7 +83,8 @@ module Shoryuken
|
|
79
83
|
:on_stop,
|
80
84
|
:on,
|
81
85
|
:cache_visibility_timeout?,
|
82
|
-
:cache_visibility_timeout
|
86
|
+
:cache_visibility_timeout=,
|
87
|
+
:delay
|
83
88
|
)
|
84
89
|
end
|
85
90
|
|
@@ -99,7 +99,7 @@ module Shoryuken
|
|
99
99
|
|
100
100
|
def prefix_active_job_queue_names
|
101
101
|
return unless Shoryuken.active_job?
|
102
|
-
return unless Shoryuken.active_job_queue_name_prefixing
|
102
|
+
return unless Shoryuken.active_job_queue_name_prefixing?
|
103
103
|
|
104
104
|
Shoryuken.options[:queues].to_a.map! do |queue_name, weight|
|
105
105
|
prefix_active_job_queue_name(queue_name, weight)
|
data/lib/shoryuken/fetcher.rb
CHANGED
@@ -14,7 +14,7 @@ module Shoryuken
|
|
14
14
|
|
15
15
|
logger.debug { "Looking for new messages in #{queue}" }
|
16
16
|
|
17
|
-
sqs_msgs = Array(receive_messages(queue,
|
17
|
+
sqs_msgs = Array(receive_messages(queue, limit))
|
18
18
|
|
19
19
|
logger.debug { "Found #{sqs_msgs.size} messages for #{queue.name}" } unless sqs_msgs.empty?
|
20
20
|
logger.debug { "Fetcher for #{queue} completed in #{elapsed(started_at)} ms" }
|
@@ -49,16 +49,7 @@ module Shoryuken
|
|
49
49
|
|
50
50
|
shoryuken_queue = Shoryuken::Client.queues(queue.name)
|
51
51
|
|
52
|
-
|
53
|
-
# if we set max_number_of_messages greater than 1,
|
54
|
-
# SQS may return more than one message for the same message group
|
55
|
-
# since Shoryuken uses threads, it will try to process more than one at once
|
56
|
-
# > The message group ID is the tag that specifies that a message belongs to a specific message group.
|
57
|
-
# > Messages that belong to the same message group are always processed one by one,
|
58
|
-
# > in a strict order relative to the message group
|
59
|
-
# > (however, messages that belong to different message groups might be processed out of order).
|
60
|
-
# > https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagegroupid-property.html
|
61
|
-
options[:max_number_of_messages] = shoryuken_queue.fifo? ? 1 : max_number_of_messages(limit, options)
|
52
|
+
options[:max_number_of_messages] = max_number_of_messages(shoryuken_queue, limit, options)
|
62
53
|
options[:message_attribute_names] = %w[All]
|
63
54
|
options[:attribute_names] = %w[All]
|
64
55
|
|
@@ -67,7 +58,18 @@ module Shoryuken
|
|
67
58
|
shoryuken_queue.receive_messages(options)
|
68
59
|
end
|
69
60
|
|
70
|
-
def max_number_of_messages(limit, options)
|
61
|
+
def max_number_of_messages(shoryuken_queue, limit, options)
|
62
|
+
# For FIFO queues we want to make sure we process one message per group at a time
|
63
|
+
# if we set max_number_of_messages greater than 1,
|
64
|
+
# SQS may return more than one message for the same message group
|
65
|
+
# since Shoryuken uses threads, it will try to process more than one message at once
|
66
|
+
# > The message group ID is the tag that specifies that a message belongs to a specific message group.
|
67
|
+
# > Messages that belong to the same message group are always processed one by one,
|
68
|
+
# > in a strict order relative to the message group
|
69
|
+
# > (however, messages that belong to different message groups might be processed out of order).
|
70
|
+
# > https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagegroupid-property.html
|
71
|
+
limit = 1 if shoryuken_queue.fifo? && !batched_queue?(shoryuken_queue)
|
72
|
+
|
71
73
|
[limit, FETCH_LIMIT, options[:max_number_of_messages]].compact.min
|
72
74
|
end
|
73
75
|
|
@@ -77,5 +79,9 @@ module Shoryuken
|
|
77
79
|
|
78
80
|
options.to_h.dup
|
79
81
|
end
|
82
|
+
|
83
|
+
def batched_queue?(queue)
|
84
|
+
Shoryuken.worker_registry.batch_receive_messages?(queue.name)
|
85
|
+
end
|
80
86
|
end
|
81
|
-
end
|
87
|
+
end
|
data/lib/shoryuken/launcher.rb
CHANGED
@@ -72,7 +72,7 @@ module Shoryuken
|
|
72
72
|
Shoryuken.groups.map do |group, options|
|
73
73
|
Shoryuken::Manager.new(
|
74
74
|
Shoryuken::Fetcher.new(group),
|
75
|
-
Shoryuken.polling_strategy(group).new(options[:queues], Shoryuken
|
75
|
+
Shoryuken.polling_strategy(group).new(options[:queues], Shoryuken.delay(group)),
|
76
76
|
options[:concurrency],
|
77
77
|
executor
|
78
78
|
)
|
data/lib/shoryuken/manager.rb
CHANGED
@@ -43,8 +43,8 @@ module Shoryuken
|
|
43
43
|
logger.debug { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{@polling_strategy.active_queues}" }
|
44
44
|
|
45
45
|
batched_queue?(queue) ? dispatch_batch(queue) : dispatch_single_messages(queue)
|
46
|
-
rescue =>
|
47
|
-
handle_dispatch_error(
|
46
|
+
rescue => e
|
47
|
+
handle_dispatch_error(e)
|
48
48
|
ensure
|
49
49
|
dispatch_loop
|
50
50
|
end
|
@@ -74,9 +74,9 @@ module Shoryuken
|
|
74
74
|
end
|
75
75
|
|
76
76
|
def dispatch_batch(queue)
|
77
|
-
|
77
|
+
batch = @fetcher.fetch(queue, BATCH_LIMIT)
|
78
78
|
@polling_strategy.messages_found(queue.name, batch.size)
|
79
|
-
assign(queue.name, patch_batch!(batch))
|
79
|
+
assign(queue.name, patch_batch!(batch)) if batch.any?
|
80
80
|
end
|
81
81
|
|
82
82
|
def dispatch_single_messages(queue)
|
data/lib/shoryuken/options.rb
CHANGED
@@ -14,228 +14,166 @@ module Shoryuken
|
|
14
14
|
}
|
15
15
|
}.freeze
|
16
16
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
defined?(::ActiveJob)
|
31
|
-
end
|
32
|
-
|
33
|
-
def add_group(group, concurrency = nil, delay: nil)
|
34
|
-
concurrency ||= options[:concurrency]
|
35
|
-
delay ||= options[:delay]
|
36
|
-
|
37
|
-
groups[group] ||= {
|
38
|
-
concurrency: concurrency,
|
39
|
-
delay: delay,
|
40
|
-
queues: []
|
41
|
-
}
|
42
|
-
end
|
43
|
-
|
44
|
-
def groups
|
45
|
-
@@groups
|
46
|
-
end
|
47
|
-
|
48
|
-
def add_queue(queue, weight, group)
|
49
|
-
weight.times do
|
50
|
-
groups[group][:queues] << queue
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
def ungrouped_queues
|
55
|
-
groups.values.flat_map { |options| options[:queues] }
|
56
|
-
end
|
57
|
-
|
58
|
-
def worker_registry
|
59
|
-
@@worker_registry
|
60
|
-
end
|
61
|
-
|
62
|
-
def worker_registry=(worker_registry)
|
63
|
-
@@worker_registry = worker_registry
|
64
|
-
end
|
65
|
-
|
66
|
-
def worker_executor
|
67
|
-
@@worker_executor
|
68
|
-
end
|
69
|
-
|
70
|
-
def worker_executor=(worker_executor)
|
71
|
-
@@worker_executor = worker_executor
|
72
|
-
end
|
73
|
-
|
74
|
-
def launcher_executor
|
75
|
-
@@launcher_executor
|
76
|
-
end
|
77
|
-
|
78
|
-
def launcher_executor=(launcher_executor)
|
79
|
-
@@launcher_executor = launcher_executor
|
80
|
-
end
|
81
|
-
|
82
|
-
def polling_strategy(group)
|
83
|
-
strategy = (group == 'default' ? options : options[:groups].to_h[group]).to_h[:polling_strategy]
|
84
|
-
case strategy
|
85
|
-
when 'WeightedRoundRobin', nil # Default case
|
86
|
-
Polling::WeightedRoundRobin
|
87
|
-
when 'StrictPriority'
|
88
|
-
Polling::StrictPriority
|
89
|
-
when Class
|
90
|
-
strategy
|
91
|
-
else
|
92
|
-
raise ArgumentError, "#{strategy} is not a valid polling_strategy"
|
93
|
-
end
|
94
|
-
end
|
95
|
-
|
96
|
-
def delay(group)
|
97
|
-
groups[group].to_h.fetch(:delay, options[:delay]).to_f
|
98
|
-
end
|
99
|
-
|
100
|
-
def start_callback
|
101
|
-
@@start_callback
|
102
|
-
end
|
103
|
-
|
104
|
-
def start_callback=(start_callback)
|
105
|
-
@@start_callback = start_callback
|
106
|
-
end
|
17
|
+
attr_accessor :active_job_queue_name_prefixing, :cache_visibility_timeout, :default_worker_options, :groups,
|
18
|
+
:launcher_executor, :sqs_client, :sqs_client_receive_message_opts,
|
19
|
+
:start_callback, :worker_executor, :worker_registry
|
20
|
+
|
21
|
+
def initialize
|
22
|
+
self.groups = {}
|
23
|
+
self.worker_registry = DefaultWorkerRegistry.new
|
24
|
+
self.active_job_queue_name_prefixing = false
|
25
|
+
self.worker_executor = Worker::DefaultExecutor
|
26
|
+
self.cache_visibility_timeout = false
|
27
|
+
# this is needed for keeping backward compatibility
|
28
|
+
@sqs_client_receive_message_opts ||= {}
|
29
|
+
end
|
107
30
|
|
108
|
-
|
109
|
-
|
110
|
-
|
31
|
+
def active_job?
|
32
|
+
defined?(::ActiveJob)
|
33
|
+
end
|
111
34
|
|
112
|
-
|
113
|
-
|
114
|
-
|
35
|
+
def add_group(group, concurrency = nil, delay: nil)
|
36
|
+
concurrency ||= options[:concurrency]
|
37
|
+
delay ||= options[:delay]
|
115
38
|
|
116
|
-
|
117
|
-
|
118
|
-
|
39
|
+
groups[group] ||= {
|
40
|
+
concurrency: concurrency,
|
41
|
+
delay: delay,
|
42
|
+
queues: []
|
43
|
+
}
|
44
|
+
end
|
119
45
|
|
120
|
-
|
121
|
-
|
46
|
+
def add_queue(queue, weight, group)
|
47
|
+
weight.times do
|
48
|
+
groups[group][:queues] << queue
|
122
49
|
end
|
50
|
+
end
|
123
51
|
|
124
|
-
|
125
|
-
|
126
|
-
|
52
|
+
def ungrouped_queues
|
53
|
+
groups.values.flat_map { |options| options[:queues] }
|
54
|
+
end
|
127
55
|
|
128
|
-
|
129
|
-
|
56
|
+
def polling_strategy(group)
|
57
|
+
strategy = (group == 'default' ? options : options[:groups].to_h[group]).to_h[:polling_strategy]
|
58
|
+
case strategy
|
59
|
+
when 'WeightedRoundRobin', nil # Default case
|
60
|
+
Polling::WeightedRoundRobin
|
61
|
+
when 'StrictPriority'
|
62
|
+
Polling::StrictPriority
|
63
|
+
when Class
|
64
|
+
strategy
|
65
|
+
else
|
66
|
+
raise ArgumentError, "#{strategy} is not a valid polling_strategy"
|
130
67
|
end
|
68
|
+
end
|
131
69
|
|
132
|
-
|
133
|
-
|
134
|
-
|
70
|
+
def delay(group)
|
71
|
+
groups[group].to_h.fetch(:delay, options[:delay]).to_f
|
72
|
+
end
|
135
73
|
|
136
|
-
|
137
|
-
|
138
|
-
|
74
|
+
def sqs_client
|
75
|
+
@sqs_client ||= Aws::SQS::Client.new
|
76
|
+
end
|
139
77
|
|
140
|
-
|
141
|
-
|
142
|
-
|
78
|
+
def sqs_client_receive_message_opts=(sqs_client_receive_message_opts)
|
79
|
+
@sqs_client_receive_message_opts['default'] = sqs_client_receive_message_opts
|
80
|
+
end
|
143
81
|
|
144
|
-
|
145
|
-
|
146
|
-
|
82
|
+
def options
|
83
|
+
@options ||= DEFAULTS.dup
|
84
|
+
end
|
147
85
|
|
148
|
-
|
149
|
-
|
150
|
-
|
86
|
+
def logger
|
87
|
+
Shoryuken::Logging.logger
|
88
|
+
end
|
151
89
|
|
152
|
-
|
153
|
-
|
154
|
-
|
90
|
+
def register_worker(*args)
|
91
|
+
worker_registry.register_worker(*args)
|
92
|
+
end
|
155
93
|
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
@@server_chain
|
160
|
-
end
|
94
|
+
def configure_server
|
95
|
+
yield self if server?
|
96
|
+
end
|
161
97
|
|
162
|
-
|
163
|
-
|
164
|
-
|
98
|
+
def server_middleware
|
99
|
+
@_server_chain ||= default_server_middleware
|
100
|
+
yield @_server_chain if block_given?
|
101
|
+
@_server_chain
|
102
|
+
end
|
165
103
|
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
@@client_chain
|
170
|
-
end
|
104
|
+
def configure_client
|
105
|
+
yield self unless server?
|
106
|
+
end
|
171
107
|
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
'auto_visibility_timeout' => false,
|
178
|
-
'retry_intervals' => nil,
|
179
|
-
'batch' => false
|
180
|
-
}
|
181
|
-
end
|
108
|
+
def client_middleware
|
109
|
+
@_client_chain ||= default_client_middleware
|
110
|
+
yield @_client_chain if block_given?
|
111
|
+
@_client_chain
|
112
|
+
end
|
182
113
|
|
183
|
-
|
184
|
-
|
185
|
-
|
114
|
+
def default_worker_options
|
115
|
+
@default_worker_options ||= {
|
116
|
+
'queue' => 'default',
|
117
|
+
'delete' => false,
|
118
|
+
'auto_delete' => false,
|
119
|
+
'auto_visibility_timeout' => false,
|
120
|
+
'retry_intervals' => nil,
|
121
|
+
'batch' => false
|
122
|
+
}
|
123
|
+
end
|
186
124
|
|
187
|
-
|
188
|
-
|
189
|
-
|
125
|
+
def on_start(&block)
|
126
|
+
self.start_callback = block
|
127
|
+
end
|
190
128
|
|
191
|
-
|
192
|
-
|
193
|
-
|
129
|
+
def on_stop(&block)
|
130
|
+
self.stop_callback = block
|
131
|
+
end
|
194
132
|
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
133
|
+
# Register a block to run at a point in the Shoryuken lifecycle.
|
134
|
+
# :startup, :quiet or :shutdown are valid events.
|
135
|
+
#
|
136
|
+
# Shoryuken.configure_server do |config|
|
137
|
+
# config.on(:shutdown) do
|
138
|
+
# puts "Goodbye cruel world!"
|
139
|
+
# end
|
140
|
+
# end
|
141
|
+
def on(event, &block)
|
142
|
+
fail ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
|
143
|
+
fail ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
|
144
|
+
|
145
|
+
options[:lifecycle_events][event] << block
|
146
|
+
end
|
208
147
|
|
209
|
-
|
210
|
-
|
211
|
-
|
148
|
+
def server?
|
149
|
+
defined?(Shoryuken::CLI)
|
150
|
+
end
|
212
151
|
|
213
|
-
|
214
|
-
|
215
|
-
|
152
|
+
def cache_visibility_timeout?
|
153
|
+
@cache_visibility_timeout
|
154
|
+
end
|
216
155
|
|
217
|
-
|
218
|
-
|
219
|
-
|
156
|
+
def active_job_queue_name_prefixing?
|
157
|
+
@active_job_queue_name_prefixing
|
158
|
+
end
|
220
159
|
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
end
|
160
|
+
private
|
161
|
+
|
162
|
+
def default_server_middleware
|
163
|
+
Middleware::Chain.new do |m|
|
164
|
+
m.add Middleware::Server::Timing
|
165
|
+
m.add Middleware::Server::ExponentialBackoffRetry
|
166
|
+
m.add Middleware::Server::AutoDelete
|
167
|
+
m.add Middleware::Server::AutoExtendVisibility
|
168
|
+
if defined?(::ActiveRecord::Base)
|
169
|
+
require 'shoryuken/middleware/server/active_record'
|
170
|
+
m.add Middleware::Server::ActiveRecord
|
233
171
|
end
|
234
172
|
end
|
173
|
+
end
|
235
174
|
|
236
|
-
|
237
|
-
|
238
|
-
end
|
175
|
+
def default_client_middleware
|
176
|
+
Middleware::Chain.new
|
239
177
|
end
|
240
178
|
end
|
241
179
|
end
|
data/lib/shoryuken/queue.rb
CHANGED
data/lib/shoryuken/version.rb
CHANGED
data/lib/shoryuken/worker.rb
CHANGED
@@ -17,9 +17,9 @@ module Shoryuken
|
|
17
17
|
alias_method :perform_at, :perform_in
|
18
18
|
|
19
19
|
def server_middleware
|
20
|
-
@
|
21
|
-
yield @
|
22
|
-
@
|
20
|
+
@_server_chain ||= Shoryuken.server_middleware.dup
|
21
|
+
yield @_server_chain if block_given?
|
22
|
+
@_server_chain
|
23
23
|
end
|
24
24
|
|
25
25
|
def shoryuken_options(opts = {})
|
data/shoryuken.gemspec
CHANGED
@@ -17,7 +17,6 @@ Gem::Specification.new do |spec|
|
|
17
17
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
18
18
|
spec.require_paths = ['lib']
|
19
19
|
|
20
|
-
spec.add_development_dependency 'bundler', '~> 1.6'
|
21
20
|
spec.add_development_dependency 'dotenv'
|
22
21
|
spec.add_development_dependency 'pry-byebug'
|
23
22
|
spec.add_development_dependency 'rake'
|
@@ -103,9 +103,9 @@ RSpec.describe Shoryuken::Fetcher do
|
|
103
103
|
|
104
104
|
context 'when FIFO' do
|
105
105
|
let(:limit) { 10 }
|
106
|
-
let(:queue) { instance_double('Shoryuken::Queue', fifo?: true) }
|
106
|
+
let(:queue) { instance_double('Shoryuken::Queue', fifo?: true, name: queue_name) }
|
107
107
|
|
108
|
-
it 'polls one message at
|
108
|
+
it 'polls one message at a time' do
|
109
109
|
# see https://github.com/phstc/shoryuken/pull/530
|
110
110
|
|
111
111
|
allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
@@ -115,6 +115,21 @@ RSpec.describe Shoryuken::Fetcher do
|
|
115
115
|
|
116
116
|
subject.fetch(queue_config, limit)
|
117
117
|
end
|
118
|
+
|
119
|
+
context 'with batch=true' do
|
120
|
+
it 'polls the provided limit' do
|
121
|
+
# see https://github.com/phstc/shoryuken/pull/530
|
122
|
+
|
123
|
+
allow(Shoryuken::Client).to receive(:queues).with(queue_name).and_return(queue)
|
124
|
+
allow(Shoryuken.worker_registry).to receive(:batch_receive_messages?).with(queue.name).and_return(true)
|
125
|
+
|
126
|
+
expect(queue).to receive(:receive_messages).with(
|
127
|
+
max_number_of_messages: limit, attribute_names: ['All'], message_attribute_names: ['All']
|
128
|
+
).and_return([])
|
129
|
+
|
130
|
+
subject.fetch(queue_config, limit)
|
131
|
+
end
|
132
|
+
end
|
118
133
|
end
|
119
134
|
end
|
120
135
|
end
|
@@ -72,11 +72,27 @@ RSpec.describe Shoryuken::Manager do
|
|
72
72
|
expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
|
73
73
|
expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
|
74
74
|
expect(Shoryuken::Processor).to receive(:process).with(q, message)
|
75
|
-
expect(Shoryuken.logger).
|
75
|
+
expect(Shoryuken.logger).to receive(:info).never
|
76
76
|
|
77
77
|
subject.send(:dispatch)
|
78
78
|
end
|
79
79
|
|
80
|
+
context 'and there are no messages in the queue' do
|
81
|
+
specify do
|
82
|
+
messages = %w[]
|
83
|
+
q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
|
84
|
+
|
85
|
+
expect(fetcher).to receive(:fetch).with(q, concurrency).and_return(messages)
|
86
|
+
expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
|
87
|
+
expect(polling_strategy).to receive(:messages_found).with(q.name, 0)
|
88
|
+
expect(Shoryuken.logger).to receive(:info).never
|
89
|
+
expect(Shoryuken::Processor).to receive(:process).never
|
90
|
+
expect_any_instance_of(described_class).to receive(:assign).never
|
91
|
+
|
92
|
+
subject.send(:dispatch)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
80
96
|
context 'when batch' do
|
81
97
|
specify do
|
82
98
|
messages = %w[test1 test2 test3]
|
@@ -86,10 +102,27 @@ RSpec.describe Shoryuken::Manager do
|
|
86
102
|
expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
|
87
103
|
allow(subject).to receive(:batched_queue?).with(q).and_return(true)
|
88
104
|
expect(Shoryuken::Processor).to receive(:process).with(q, messages)
|
89
|
-
expect(Shoryuken.logger).
|
105
|
+
expect(Shoryuken.logger).to receive(:info).never
|
90
106
|
|
91
107
|
subject.send(:dispatch)
|
92
108
|
end
|
109
|
+
|
110
|
+
context 'and there are no messages in the queue' do
|
111
|
+
specify do
|
112
|
+
messages = %w[]
|
113
|
+
q = Shoryuken::Polling::QueueConfiguration.new(queue, {})
|
114
|
+
|
115
|
+
expect(fetcher).to receive(:fetch).with(q, described_class::BATCH_LIMIT).and_return(messages)
|
116
|
+
expect(subject).to receive(:fire_event).with(:dispatch, false, queue_name: q.name)
|
117
|
+
allow(subject).to receive(:batched_queue?).with(q).and_return(true)
|
118
|
+
expect(polling_strategy).to receive(:messages_found).with(q.name, 0)
|
119
|
+
expect(Shoryuken.logger).to receive(:info).never
|
120
|
+
expect(Shoryuken::Processor).to receive(:process).never
|
121
|
+
expect_any_instance_of(described_class).to receive(:assign).never
|
122
|
+
|
123
|
+
subject.send(:dispatch)
|
124
|
+
end
|
125
|
+
end
|
93
126
|
end
|
94
127
|
end
|
95
128
|
|
@@ -1,6 +1,8 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
RSpec.describe Shoryuken::Options do
|
4
|
+
subject { Shoryuken.shoryuken_options }
|
5
|
+
|
4
6
|
describe '.add_group adds queues and optional delay' do
|
5
7
|
before do
|
6
8
|
Shoryuken.groups.clear
|
@@ -10,14 +12,14 @@ RSpec.describe Shoryuken::Options do
|
|
10
12
|
end
|
11
13
|
|
12
14
|
specify do
|
13
|
-
|
14
|
-
|
15
|
-
|
15
|
+
subject.add_queue('queue1', 1, 'group1')
|
16
|
+
subject.add_queue('queue2', 2, 'group2')
|
17
|
+
subject.add_queue('queue3', 1, 'group3')
|
16
18
|
|
17
|
-
expect(
|
18
|
-
expect(
|
19
|
-
expect(
|
20
|
-
expect(
|
19
|
+
expect(subject.groups['group1'][:queues]).to eq(%w[queue1])
|
20
|
+
expect(subject.groups['group2'][:queues]).to eq(%w[queue2 queue2])
|
21
|
+
expect(subject.groups['group3'][:queues]).to eq(%w[queue3])
|
22
|
+
expect(subject.groups['group3'][:delay]).to eq(5)
|
21
23
|
end
|
22
24
|
end
|
23
25
|
|
@@ -25,11 +27,11 @@ RSpec.describe Shoryuken::Options do
|
|
25
27
|
specify do
|
26
28
|
Shoryuken.add_group('group1', 25)
|
27
29
|
Shoryuken.add_group('group2', 25, delay: 5)
|
28
|
-
|
29
|
-
|
30
|
+
subject.add_queue('queue1', 1, 'group1')
|
31
|
+
subject.add_queue('queue2', 2, 'group2')
|
30
32
|
|
31
|
-
expect(
|
32
|
-
expect(
|
33
|
+
expect(subject.delay('group1')).to eq(Shoryuken.options[:delay])
|
34
|
+
expect(subject.delay('group2')).to eq(5.0)
|
33
35
|
end
|
34
36
|
end
|
35
37
|
|
@@ -41,10 +43,10 @@ RSpec.describe Shoryuken::Options do
|
|
41
43
|
end
|
42
44
|
|
43
45
|
specify do
|
44
|
-
|
45
|
-
|
46
|
+
subject.add_queue('queue1', 1, 'group1')
|
47
|
+
subject.add_queue('queue2', 2, 'group2')
|
46
48
|
|
47
|
-
expect(
|
49
|
+
expect(subject.ungrouped_queues).to eq(%w[queue1 queue2 queue2])
|
48
50
|
end
|
49
51
|
end
|
50
52
|
|
@@ -68,22 +70,22 @@ RSpec.describe Shoryuken::Options do
|
|
68
70
|
|
69
71
|
describe '.register_worker' do
|
70
72
|
it 'registers a worker' do
|
71
|
-
|
72
|
-
|
73
|
-
expect(
|
73
|
+
subject.worker_registry.clear
|
74
|
+
subject.register_worker('default', TestWorker)
|
75
|
+
expect(subject.worker_registry.workers('default')).to eq([TestWorker])
|
74
76
|
end
|
75
77
|
|
76
78
|
it 'registers a batchable worker' do
|
77
|
-
|
79
|
+
subject.worker_registry.clear
|
78
80
|
TestWorker.get_shoryuken_options['batch'] = true
|
79
|
-
|
80
|
-
expect(
|
81
|
+
subject.register_worker('default', TestWorker)
|
82
|
+
expect(subject.worker_registry.workers('default')).to eq([TestWorker])
|
81
83
|
end
|
82
84
|
|
83
85
|
it 'allows multiple workers' do
|
84
|
-
|
85
|
-
|
86
|
-
expect(
|
86
|
+
subject.worker_registry.clear
|
87
|
+
subject.register_worker('default', TestWorker)
|
88
|
+
expect(subject.worker_registry.workers('default')).to eq([TestWorker])
|
87
89
|
|
88
90
|
class Test2Worker
|
89
91
|
include Shoryuken::Worker
|
@@ -93,13 +95,13 @@ RSpec.describe Shoryuken::Options do
|
|
93
95
|
def perform(sqs_msg, body); end
|
94
96
|
end
|
95
97
|
|
96
|
-
expect(
|
98
|
+
expect(subject.worker_registry.workers('default')).to eq([Test2Worker])
|
97
99
|
end
|
98
100
|
|
99
101
|
it 'raises an exception when mixing batchable with non batchable' do
|
100
|
-
|
102
|
+
subject.worker_registry.clear
|
101
103
|
TestWorker.get_shoryuken_options['batch'] = true
|
102
|
-
|
104
|
+
subject.register_worker('default', TestWorker)
|
103
105
|
|
104
106
|
expect {
|
105
107
|
class BatchableWorker
|
@@ -57,8 +57,8 @@ RSpec.describe Shoryuken::Processor do
|
|
57
57
|
|
58
58
|
context 'server' do
|
59
59
|
before do
|
60
|
-
|
61
|
-
WorkerCalledMiddlewareWorker.instance_variable_set(:@
|
60
|
+
allow_any_instance_of(Shoryuken::Options).to receive(:server?).and_return(true)
|
61
|
+
WorkerCalledMiddlewareWorker.instance_variable_set(:@_server_chain, nil) # un-memoize middleware
|
62
62
|
|
63
63
|
Shoryuken.configure_server do |config|
|
64
64
|
config.server_middleware do |chain|
|
@@ -85,8 +85,8 @@ RSpec.describe Shoryuken::Processor do
|
|
85
85
|
|
86
86
|
context 'client' do
|
87
87
|
before do
|
88
|
-
|
89
|
-
WorkerCalledMiddlewareWorker.instance_variable_set(:@
|
88
|
+
allow_any_instance_of(Shoryuken::Options).to receive(:server?).and_return(false)
|
89
|
+
WorkerCalledMiddlewareWorker.instance_variable_set(:@_server_chain, nil) # un-memoize middleware
|
90
90
|
|
91
91
|
Shoryuken.configure_server do |config|
|
92
92
|
config.server_middleware do |chain|
|
@@ -26,6 +26,19 @@ RSpec.describe Shoryuken::Queue do
|
|
26
26
|
end
|
27
27
|
end
|
28
28
|
|
29
|
+
context 'when a non SQS queue URL' do
|
30
|
+
let(:queue_url) { "http://localhost:4576/queue/#{queue_name}" }
|
31
|
+
|
32
|
+
it 'instantiates by URL and validate the URL' do
|
33
|
+
# See https://github.com/phstc/shoryuken/pull/551
|
34
|
+
expect_any_instance_of(described_class).to receive(:fifo?).and_return(false)
|
35
|
+
|
36
|
+
subject = described_class.new(sqs, queue_url)
|
37
|
+
|
38
|
+
expect(subject.name).to eq(queue_name)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
29
42
|
context 'when queue name supplied' do
|
30
43
|
subject { described_class.new(sqs, queue_name) }
|
31
44
|
|
metadata
CHANGED
@@ -1,29 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shoryuken
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 5.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Pablo Cantero
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-06-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
-
- !ruby/object:Gem::Dependency
|
14
|
-
name: bundler
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - "~>"
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '1.6'
|
20
|
-
type: :development
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - "~>"
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: '1.6'
|
27
13
|
- !ruby/object:Gem::Dependency
|
28
14
|
name: dotenv
|
29
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -227,7 +213,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
227
213
|
- !ruby/object:Gem::Version
|
228
214
|
version: '0'
|
229
215
|
requirements: []
|
230
|
-
|
216
|
+
rubyforge_project:
|
217
|
+
rubygems_version: 2.7.6.2
|
231
218
|
signing_key:
|
232
219
|
specification_version: 4
|
233
220
|
summary: Shoryuken is a super efficient AWS SQS thread based message processor
|