shoryuken 1.0.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.hound.yml +6 -2
- data/.rubocop.yml +50 -0
- data/CHANGELOG.md +11 -0
- data/Gemfile +4 -1
- data/README.md +6 -52
- data/examples/bootstrap_queues.rb +1 -1
- data/lib/shoryuken/cli.rb +14 -14
- data/lib/shoryuken/client.rb +4 -7
- data/lib/shoryuken/default_worker_registry.rb +1 -1
- data/lib/shoryuken/environment_loader.rb +34 -16
- data/lib/shoryuken/extensions/active_job_adapter.rb +0 -1
- data/lib/shoryuken/fetcher.rb +8 -6
- data/lib/shoryuken/launcher.rb +1 -1
- data/lib/shoryuken/logging.rb +4 -6
- data/lib/shoryuken/manager.rb +13 -13
- data/lib/shoryuken/message.rb +70 -0
- data/lib/shoryuken/middleware/chain.rb +2 -2
- data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +50 -0
- data/lib/shoryuken/middleware/server/timing.rb +4 -4
- data/lib/shoryuken/processor.rb +28 -14
- data/lib/shoryuken/queue.rb +84 -0
- data/lib/shoryuken/sns_arn.rb +3 -3
- data/lib/shoryuken/util.rb +5 -6
- data/lib/shoryuken/version.rb +1 -1
- data/lib/shoryuken/worker.rb +1 -2
- data/lib/shoryuken.rb +29 -7
- data/shoryuken.gemspec +15 -16
- data/spec/shoryuken/client_spec.rb +45 -4
- data/spec/shoryuken/default_worker_registry_spec.rb +1 -1
- data/spec/shoryuken/fetcher_spec.rb +6 -6
- data/spec/shoryuken/middleware/chain_spec.rb +3 -1
- data/spec/shoryuken/middleware/server/auto_delete_spec.rb +2 -2
- data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +77 -0
- data/spec/shoryuken/middleware/server/timing_spec.rb +23 -9
- data/spec/shoryuken/processor_spec.rb +39 -13
- data/spec/shoryuken/queue_spec.rb +100 -0
- data/spec/shoryuken/util_spec.rb +1 -1
- data/spec/shoryuken/worker_spec.rb +0 -13
- data/spec/shoryuken_endpoint.yml +6 -0
- data/spec/spec_helper.rb +4 -4
- metadata +36 -39
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ee01f5dc2129537f640aba700e43c8be0f37e77c
|
|
4
|
+
data.tar.gz: 54c1fe1a997a29a77939336ca5f8f2c7e691a745
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 5543a21bcfadc457a3bb7444b27ba97d59c9f238bbc9633a100555154823132f7ee6f3cf233f84743ed553cc69a27f8ddf836515690a120b291ffb8212db8783
|
|
7
|
+
data.tar.gz: bcdc762ee9e02366ad4194e372ad76e2e7c23f7e5f15129d8c90d6abfa65af337c4900addaf47d14f8f6578e084a6fd84dad478878589e0c3e2912c5851c89f5
|
data/.gitignore
CHANGED
data/.hound.yml
CHANGED
data/.rubocop.yml
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
LineLength:
|
|
2
|
+
Max: 120
|
|
3
|
+
|
|
4
|
+
Style/SignalException:
|
|
5
|
+
Enabled: false
|
|
6
|
+
|
|
7
|
+
Style/SpaceAroundEqualsInParameterDefault:
|
|
8
|
+
Enabled: false
|
|
9
|
+
|
|
10
|
+
Style/Documentation:
|
|
11
|
+
Enabled: false
|
|
12
|
+
|
|
13
|
+
Style/ClassAndModuleChildren:
|
|
14
|
+
Enabled: false
|
|
15
|
+
|
|
16
|
+
Metrics/PerceivedComplexity:
|
|
17
|
+
Enabled: false
|
|
18
|
+
|
|
19
|
+
Metrics/CyclomaticComplexity:
|
|
20
|
+
Enabled: false
|
|
21
|
+
|
|
22
|
+
Style/CommentAnnotation:
|
|
23
|
+
Enabled: false
|
|
24
|
+
|
|
25
|
+
Metrics/ClassLength:
|
|
26
|
+
Enabled: false
|
|
27
|
+
|
|
28
|
+
Metrics/ParameterLists:
|
|
29
|
+
Enabled: false
|
|
30
|
+
|
|
31
|
+
Metrics/MethodLength :
|
|
32
|
+
Enabled: false
|
|
33
|
+
|
|
34
|
+
Style/PerlBackrefs:
|
|
35
|
+
Enabled: false
|
|
36
|
+
|
|
37
|
+
Style/StringLiterals:
|
|
38
|
+
EnforcedStyle: single_quotes
|
|
39
|
+
SupportedStyles:
|
|
40
|
+
- single_quotes
|
|
41
|
+
- double_quotes
|
|
42
|
+
|
|
43
|
+
Style/StringLiteralsInInterpolation:
|
|
44
|
+
EnforcedStyle: single_quotes
|
|
45
|
+
SupportedStyles:
|
|
46
|
+
- single_quotes
|
|
47
|
+
- double_quotes
|
|
48
|
+
|
|
49
|
+
Lint/UnusedMethodArgument:
|
|
50
|
+
Enabled: false
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
## [v2.0.0] - 2015-09-22
|
|
2
|
+
|
|
3
|
+
- Allow configuration of SQS/SNS endpoints via environment variables
|
|
4
|
+
- [#130](https://github.com/phstc/shoryuken/pull/130)
|
|
5
|
+
|
|
6
|
+
- Expose queue_name in the message object
|
|
7
|
+
- [#127](https://github.com/phstc/shoryuken/pull/127)
|
|
8
|
+
|
|
9
|
+
- README updates
|
|
10
|
+
- [#122](https://github.com/phstc/shoryuken/pull/122)
|
|
11
|
+
- [#120](https://github.com/phstc/shoryuken/pull/120)
|
data/Gemfile
CHANGED
data/README.md
CHANGED
|
@@ -36,7 +36,7 @@ If all queues get empty, all processors will be changed to the waiting state and
|
|
|
36
36
|
|
|
37
37
|
### Fetch in batches
|
|
38
38
|
|
|
39
|
-
To be even more
|
|
39
|
+
To be even more performant and cost effective, Shoryuken fetches SQS messages in batches, so a single SQS request can fetch up to 10 messages.
|
|
40
40
|
|
|
41
41
|
## Requirements
|
|
42
42
|
|
|
@@ -114,7 +114,7 @@ aws:
|
|
|
114
114
|
access_key_id: ... # or <%= ENV['AWS_ACCESS_KEY_ID'] %>
|
|
115
115
|
secret_access_key: ... # or <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
|
|
116
116
|
region: us-east-1 # or <%= ENV['AWS_REGION'] %>
|
|
117
|
-
receive_message: # See http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/
|
|
117
|
+
receive_message: # See http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#receive_message-instance_method
|
|
118
118
|
# wait_time_seconds: N # The number of seconds to wait for new messages when polling. Defaults to the #wait_time_seconds defined on the queue
|
|
119
119
|
attribute_names:
|
|
120
120
|
- ApproximateReceiveCount
|
|
@@ -132,59 +132,13 @@ The ```aws``` section is used to configure both the Aws objects used by Shoryuke
|
|
|
132
132
|
- ```account_id``` is used when generating SNS ARNs
|
|
133
133
|
- ```sns_endpoint``` can be used to explicitly override the SNS endpoint
|
|
134
134
|
- ```sqs_endpoint``` can be used to explicitly override the SQS endpoint
|
|
135
|
-
- ```receive_message``` can be used to define the options passed to the http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/
|
|
135
|
+
- ```receive_message``` can be used to define the options passed to the http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#receive_message-instance_method
|
|
136
136
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
You can tell Shoryuken to load your Rails application by passing the `-R` or `--rails` flag to the "shoryuken" command.
|
|
140
|
-
|
|
141
|
-
If you load Rails, and assuming your workers are located in the `app/workers` directory, they will be auto-loaded. This means you don't need to require them explicitly with `-r`.
|
|
142
|
-
|
|
143
|
-
For middleware and other configuration, you might want to create an initializer:
|
|
144
|
-
|
|
145
|
-
```ruby
|
|
146
|
-
Shoryuken.configure_server do |config|
|
|
147
|
-
# Replace Rails logger so messages are logged wherever Shoryuken is logging
|
|
148
|
-
# Note: this entire block is only run by the processor, so we don't overwrite
|
|
149
|
-
# the logger when the app is running as usual.
|
|
150
|
-
Rails.logger = Shoryuken::Logging.logger
|
|
151
|
-
|
|
152
|
-
config.server_middleware do |chain|
|
|
153
|
-
chain.add Shoryuken::MyMiddleware
|
|
154
|
-
end
|
|
155
|
-
end
|
|
156
|
-
```
|
|
157
|
-
|
|
158
|
-
*Note:* In the above case, since we are replacing the Rails logger, it's desired that this initializer runs before other initializers (in case they themselves use the logger). Since by Rails conventions initializers are executed in alphabetical order, this can be achieved by prepending the initializer filename with `00_` (assuming no other initializers alphabetically precede this one).
|
|
159
|
-
|
|
160
|
-
This feature works for Rails 4+, but needs to be confirmed for older versions.
|
|
161
|
-
|
|
162
|
-
#### ActiveJob Support
|
|
137
|
+
The ```sns_endpoint``` and ```sqs_endpoint``` Shoryuken-specific options will also fallback to the environment variables ```AWS_SNS_ENDPOINT``` and ```AWS_SQS_ENDPOINT``` respectively, if they are set.
|
|
163
138
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
It works as expected. Just put your job in `app/jobs`. Here's an example:
|
|
167
|
-
|
|
168
|
-
```ruby
|
|
169
|
-
class ProcessPhotoJob < ActiveJob::Base
|
|
170
|
-
queue_as :default
|
|
171
|
-
|
|
172
|
-
rescue_from ActiveJob::DeserializationError do |e|
|
|
173
|
-
Shoryuken.logger.error ex
|
|
174
|
-
Shoryuken.logger.error ex.backtrace.join("\n")
|
|
175
|
-
end
|
|
176
|
-
|
|
177
|
-
def perform(photo)
|
|
178
|
-
photo.process_image!
|
|
179
|
-
end
|
|
180
|
-
end
|
|
181
|
-
```
|
|
182
|
-
|
|
183
|
-
Delayed mailers, ActiveRecord serialization, etc. all work.
|
|
184
|
-
|
|
185
|
-
See [ActiveJob docs](http://edgeguides.rubyonrails.org/active_job_basics.html) for more info.
|
|
139
|
+
### Rails Integration
|
|
186
140
|
|
|
187
|
-
|
|
141
|
+
[Check the Rails Integration Active Job documention](https://github.com/phstc/shoryuken/wiki/Rails-Integration-Active-Job).
|
|
188
142
|
|
|
189
143
|
### Start Shoryuken
|
|
190
144
|
|
|
@@ -22,7 +22,7 @@ if sqs.config['endpoint'] =~ /amazonaws.com/
|
|
|
22
22
|
).attributes['QueueArn']
|
|
23
23
|
|
|
24
24
|
attributes = {}
|
|
25
|
-
attributes['RedrivePolicy'] = %Q{
|
|
25
|
+
attributes['RedrivePolicy'] = %Q({"maxReceiveCount":"7", "deadLetterTargetArn":"#{dead_letter_queue_arn}"})
|
|
26
26
|
|
|
27
27
|
sqs.set_queue_attributes queue_url: default_queue_url, attributes: attributes
|
|
28
28
|
end
|
data/lib/shoryuken/cli.rb
CHANGED
|
@@ -19,7 +19,7 @@ module Shoryuken
|
|
|
19
19
|
def run(args)
|
|
20
20
|
self_read, self_write = IO.pipe
|
|
21
21
|
|
|
22
|
-
%w
|
|
22
|
+
%w(INT TERM USR1 USR2 TTIN).each do |sig|
|
|
23
23
|
begin
|
|
24
24
|
trap sig do
|
|
25
25
|
self_write.puts(sig)
|
|
@@ -41,14 +41,14 @@ module Shoryuken
|
|
|
41
41
|
@launcher = Shoryuken::Launcher.new
|
|
42
42
|
|
|
43
43
|
if callback = Shoryuken.start_callback
|
|
44
|
-
logger.info
|
|
44
|
+
logger.info { 'Calling Shoryuken.on_start block' }
|
|
45
45
|
callback.call
|
|
46
46
|
end
|
|
47
47
|
|
|
48
48
|
begin
|
|
49
49
|
launcher.run
|
|
50
50
|
|
|
51
|
-
while readable_io = IO.select([self_read])
|
|
51
|
+
while (readable_io = IO.select([self_read]))
|
|
52
52
|
signal = readable_io.first[0].gets.strip
|
|
53
53
|
handle_signal(signal)
|
|
54
54
|
end
|
|
@@ -86,7 +86,7 @@ module Shoryuken
|
|
|
86
86
|
|
|
87
87
|
files_to_reopen.each do |file|
|
|
88
88
|
begin
|
|
89
|
-
file.reopen file.path,
|
|
89
|
+
file.reopen file.path, 'a+'
|
|
90
90
|
file.sync = true
|
|
91
91
|
rescue ::Exception
|
|
92
92
|
end
|
|
@@ -102,7 +102,7 @@ module Shoryuken
|
|
|
102
102
|
end
|
|
103
103
|
|
|
104
104
|
def write_pid
|
|
105
|
-
if path = Shoryuken.options[:pidfile]
|
|
105
|
+
if (path = Shoryuken.options[:pidfile])
|
|
106
106
|
File.open(path, 'w') do |f|
|
|
107
107
|
f.puts Process.pid
|
|
108
108
|
end
|
|
@@ -150,7 +150,7 @@ module Shoryuken
|
|
|
150
150
|
opts[:verbose] = arg
|
|
151
151
|
end
|
|
152
152
|
|
|
153
|
-
o.on '-V', '--version', 'Print version and exit' do
|
|
153
|
+
o.on '-V', '--version', 'Print version and exit' do
|
|
154
154
|
puts "Shoryuken #{Shoryuken::VERSION}"
|
|
155
155
|
exit 0
|
|
156
156
|
end
|
|
@@ -158,7 +158,7 @@ module Shoryuken
|
|
|
158
158
|
|
|
159
159
|
@parser.banner = 'shoryuken [options]'
|
|
160
160
|
@parser.on_tail '-h', '--help', 'Show help' do
|
|
161
|
-
logger.info @parser
|
|
161
|
+
logger.info { @parser }
|
|
162
162
|
exit 1
|
|
163
163
|
end
|
|
164
164
|
@parser.parse!(argv)
|
|
@@ -166,22 +166,22 @@ module Shoryuken
|
|
|
166
166
|
end
|
|
167
167
|
|
|
168
168
|
def handle_signal(sig)
|
|
169
|
-
logger.info "Got #{sig} signal"
|
|
169
|
+
logger.info { "Got #{sig} signal" }
|
|
170
170
|
|
|
171
171
|
case sig
|
|
172
172
|
when 'USR1'
|
|
173
|
-
logger.info
|
|
173
|
+
logger.info { 'Received USR1, will soft shutdown down' }
|
|
174
174
|
|
|
175
175
|
launcher.stop
|
|
176
176
|
|
|
177
177
|
exit 0
|
|
178
178
|
when 'TTIN'
|
|
179
179
|
Thread.list.each do |thread|
|
|
180
|
-
logger.info "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
|
|
180
|
+
logger.info { "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" }
|
|
181
181
|
if thread.backtrace
|
|
182
|
-
logger.info thread.backtrace.join("\n")
|
|
182
|
+
logger.info { thread.backtrace.join("\n") }
|
|
183
183
|
else
|
|
184
|
-
logger.info
|
|
184
|
+
logger.info { '<no backtrace available>' }
|
|
185
185
|
end
|
|
186
186
|
end
|
|
187
187
|
|
|
@@ -189,9 +189,9 @@ module Shoryuken
|
|
|
189
189
|
busy = launcher.manager.instance_variable_get(:@busy).size
|
|
190
190
|
queues = launcher.manager.instance_variable_get(:@queues)
|
|
191
191
|
|
|
192
|
-
logger.info "Ready: #{ready}, Busy: #{busy}, Active Queues: #{unparse_queues(queues)}"
|
|
192
|
+
logger.info { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{unparse_queues(queues)}" }
|
|
193
193
|
else
|
|
194
|
-
logger.info "Received #{sig}, will shutdown down"
|
|
194
|
+
logger.info { "Received #{sig}, will shutdown down" }
|
|
195
195
|
|
|
196
196
|
raise Interrupt
|
|
197
197
|
end
|
data/lib/shoryuken/client.rb
CHANGED
|
@@ -5,7 +5,7 @@ module Shoryuken
|
|
|
5
5
|
|
|
6
6
|
class << self
|
|
7
7
|
def queues(name)
|
|
8
|
-
@@queues[name.to_s] ||=
|
|
8
|
+
@@queues[name.to_s] ||= Shoryuken::Queue.new(sqs, name)
|
|
9
9
|
end
|
|
10
10
|
|
|
11
11
|
def sns
|
|
@@ -20,10 +20,6 @@ module Shoryuken
|
|
|
20
20
|
@sqs ||= Aws::SQS::Client.new(aws_client_options(:sqs_endpoint))
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
-
def sqs_resource
|
|
24
|
-
@sqs_resource ||= Aws::SQS::Resource.new(client: sqs)
|
|
25
|
-
end
|
|
26
|
-
|
|
27
23
|
def topics(name)
|
|
28
24
|
@@topics[name.to_s] ||= Topic.new(name, sns)
|
|
29
25
|
end
|
|
@@ -33,8 +29,9 @@ module Shoryuken
|
|
|
33
29
|
|
|
34
30
|
private
|
|
35
31
|
|
|
36
|
-
def aws_client_options
|
|
37
|
-
|
|
32
|
+
def aws_client_options(service_endpoint_key)
|
|
33
|
+
environment_endpoint = ENV["AWS_#{service_endpoint_key.to_s.upcase}"]
|
|
34
|
+
explicit_endpoint = Shoryuken.options[:aws][service_endpoint_key] || environment_endpoint
|
|
38
35
|
options = {}
|
|
39
36
|
options[:endpoint] = explicit_endpoint unless explicit_endpoint.to_s.empty?
|
|
40
37
|
options
|
|
@@ -28,7 +28,7 @@ module Shoryuken
|
|
|
28
28
|
end
|
|
29
29
|
|
|
30
30
|
def register_worker(queue, clazz)
|
|
31
|
-
if worker_class = @workers[queue]
|
|
31
|
+
if (worker_class = @workers[queue])
|
|
32
32
|
if worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true
|
|
33
33
|
raise ArgumentError, "Could not register #{clazz} for '#{queue}', "\
|
|
34
34
|
"because #{worker_class} is already registered for this queue, "\
|
|
@@ -10,7 +10,7 @@ module Shoryuken
|
|
|
10
10
|
load(config_file: (Rails.root + 'config' + 'shoryuken.yml'))
|
|
11
11
|
end
|
|
12
12
|
|
|
13
|
-
def initialize
|
|
13
|
+
def initialize(options)
|
|
14
14
|
@options = options
|
|
15
15
|
end
|
|
16
16
|
|
|
@@ -19,6 +19,7 @@ module Shoryuken
|
|
|
19
19
|
load_rails if options[:rails]
|
|
20
20
|
Shoryuken.options.merge!(config_file_options)
|
|
21
21
|
merge_cli_defined_queues
|
|
22
|
+
prefix_active_job_queue_names
|
|
22
23
|
Shoryuken.options.merge!(options)
|
|
23
24
|
parse_queues
|
|
24
25
|
require_workers
|
|
@@ -33,7 +34,7 @@ module Shoryuken
|
|
|
33
34
|
def config_file_options
|
|
34
35
|
if (path = options[:config_file])
|
|
35
36
|
unless File.exist?(path)
|
|
36
|
-
Shoryuken.logger.warn "Config file #{path} does not exist"
|
|
37
|
+
Shoryuken.logger.warn { "Config file #{path} does not exist" }
|
|
37
38
|
path = nil
|
|
38
39
|
end
|
|
39
40
|
end
|
|
@@ -48,24 +49,26 @@ module Shoryuken
|
|
|
48
49
|
# when not explicit supplied
|
|
49
50
|
return if Shoryuken.options[:aws].empty?
|
|
50
51
|
|
|
51
|
-
shoryuken_keys = %
|
|
52
|
+
shoryuken_keys = %w(
|
|
52
53
|
account_id
|
|
53
54
|
sns_endpoint
|
|
54
55
|
sqs_endpoint
|
|
55
|
-
receive_message)
|
|
56
|
+
receive_message).map(&:to_sym)
|
|
56
57
|
|
|
57
58
|
aws_options = Shoryuken.options[:aws].reject do |k, v|
|
|
58
59
|
shoryuken_keys.include?(k)
|
|
59
60
|
end
|
|
60
61
|
|
|
62
|
+
# assume credentials based authentication
|
|
61
63
|
credentials = Aws::Credentials.new(
|
|
62
64
|
aws_options.delete(:access_key_id),
|
|
63
65
|
aws_options.delete(:secret_access_key))
|
|
64
66
|
|
|
65
|
-
|
|
67
|
+
# but only if the configuration options have valid values
|
|
68
|
+
aws_options = aws_options.merge(credentials: credentials) if credentials.set?
|
|
66
69
|
|
|
67
|
-
if callback = Shoryuken.aws_initialization_callback
|
|
68
|
-
Shoryuken.logger.info
|
|
70
|
+
if (callback = Shoryuken.aws_initialization_callback)
|
|
71
|
+
Shoryuken.logger.info { 'Calling Shoryuken.on_aws_initialization block' }
|
|
69
72
|
callback.call(aws_options)
|
|
70
73
|
end
|
|
71
74
|
|
|
@@ -82,19 +85,19 @@ module Shoryuken
|
|
|
82
85
|
|
|
83
86
|
require 'rails'
|
|
84
87
|
if ::Rails::VERSION::MAJOR < 4
|
|
85
|
-
require File.expand_path(
|
|
88
|
+
require File.expand_path('config/environment.rb')
|
|
86
89
|
::Rails.application.eager_load!
|
|
87
90
|
else
|
|
88
91
|
# Painful contortions, see 1791 for discussion
|
|
89
|
-
require File.expand_path(
|
|
90
|
-
::Rails::Application.initializer
|
|
92
|
+
require File.expand_path('config/application.rb')
|
|
93
|
+
::Rails::Application.initializer 'shoryuken.eager_load' do
|
|
91
94
|
::Rails.application.config.eager_load = true
|
|
92
95
|
end
|
|
93
96
|
require 'shoryuken/extensions/active_job_adapter' if defined?(::ActiveJob)
|
|
94
|
-
require File.expand_path(
|
|
97
|
+
require File.expand_path('config/environment.rb')
|
|
95
98
|
end
|
|
96
99
|
|
|
97
|
-
Shoryuken.logger.info
|
|
100
|
+
Shoryuken.logger.info { 'Rails environment loaded' }
|
|
98
101
|
end
|
|
99
102
|
|
|
100
103
|
def merge_cli_defined_queues
|
|
@@ -109,6 +112,21 @@ module Shoryuken
|
|
|
109
112
|
end
|
|
110
113
|
end
|
|
111
114
|
|
|
115
|
+
def prefix_active_job_queue_names
|
|
116
|
+
return unless @options[:rails]
|
|
117
|
+
return unless Shoryuken.active_job_queue_name_prefixing
|
|
118
|
+
|
|
119
|
+
queue_name_prefix = ::ActiveJob::Base.queue_name_prefix
|
|
120
|
+
queue_name_delimiter = ::ActiveJob::Base.queue_name_delimiter
|
|
121
|
+
|
|
122
|
+
# See https://github.com/rails/rails/blob/master/activejob/lib/active_job/queue_name.rb#L27
|
|
123
|
+
Shoryuken.options[:queues].to_a.map! do |queue_name, weight|
|
|
124
|
+
name_parts = [queue_name_prefix.presence, queue_name]
|
|
125
|
+
prefixed_queue_name = name_parts.compact.join(queue_name_delimiter)
|
|
126
|
+
[prefixed_queue_name, weight]
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
112
130
|
def parse_queue(queue, weight = nil)
|
|
113
131
|
[weight.to_i, 1].max.times { Shoryuken.queues << queue }
|
|
114
132
|
end
|
|
@@ -123,7 +141,7 @@ module Shoryuken
|
|
|
123
141
|
Shoryuken.worker_registry.queues.each do |queue|
|
|
124
142
|
Shoryuken.worker_registry.workers(queue).each do |worker_class|
|
|
125
143
|
if worker_class.instance_method(:perform).arity == 1
|
|
126
|
-
Shoryuken.logger.warn "[DEPRECATION] #{worker_class.name}#perform(sqs_msg) is deprecated. Please use #{worker_class.name}#perform(sqs_msg, body)"
|
|
144
|
+
Shoryuken.logger.warn { "[DEPRECATION] #{worker_class.name}#perform(sqs_msg) is deprecated. Please use #{worker_class.name}#perform(sqs_msg, body)" }
|
|
127
145
|
|
|
128
146
|
worker_class.class_eval do
|
|
129
147
|
alias_method :deprecated_perform, :perform
|
|
@@ -142,13 +160,13 @@ module Shoryuken
|
|
|
142
160
|
end
|
|
143
161
|
|
|
144
162
|
def validate_queues
|
|
145
|
-
Shoryuken.logger.warn 'No queues supplied' if Shoryuken.queues.empty?
|
|
163
|
+
Shoryuken.logger.warn { 'No queues supplied' } if Shoryuken.queues.empty?
|
|
146
164
|
|
|
147
165
|
Shoryuken.queues.uniq.each do |queue|
|
|
148
166
|
begin
|
|
149
167
|
Shoryuken::Client.queues queue
|
|
150
168
|
rescue Aws::SQS::Errors::NonExistentQueue
|
|
151
|
-
Shoryuken.logger.warn "AWS Queue '#{queue}' does not exist"
|
|
169
|
+
Shoryuken.logger.warn { "AWS Queue '#{queue}' does not exist" }
|
|
152
170
|
end
|
|
153
171
|
end
|
|
154
172
|
end
|
|
@@ -159,7 +177,7 @@ module Shoryuken
|
|
|
159
177
|
|
|
160
178
|
unless defined?(::ActiveJob)
|
|
161
179
|
(all_queues - queues_with_workers).each do |queue|
|
|
162
|
-
Shoryuken.logger.warn "No worker supplied for '#{queue}'"
|
|
180
|
+
Shoryuken.logger.warn { "No worker supplied for '#{queue}'" }
|
|
163
181
|
end
|
|
164
182
|
end
|
|
165
183
|
end
|
data/lib/shoryuken/fetcher.rb
CHANGED
|
@@ -13,9 +13,10 @@ module Shoryuken
|
|
|
13
13
|
# AWS limits the batch size by 10
|
|
14
14
|
limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
|
|
15
15
|
|
|
16
|
-
options = Shoryuken.options[:aws][:receive_message].
|
|
16
|
+
options = (Shoryuken.options[:aws][:receive_message] || {}).dup
|
|
17
17
|
options[:max_number_of_messages] = limit
|
|
18
18
|
options[:message_attribute_names] = %w(All)
|
|
19
|
+
options[:attribute_names] = %w(All)
|
|
19
20
|
|
|
20
21
|
Shoryuken::Client.queues(queue).receive_messages options
|
|
21
22
|
end
|
|
@@ -24,14 +25,14 @@ module Shoryuken
|
|
|
24
25
|
watchdog('Fetcher#fetch died') do
|
|
25
26
|
started_at = Time.now
|
|
26
27
|
|
|
27
|
-
logger.debug "Looking for new messages in '#{queue}'"
|
|
28
|
+
logger.debug { "Looking for new messages in '#{queue}'" }
|
|
28
29
|
|
|
29
30
|
begin
|
|
30
31
|
batch = Shoryuken.worker_registry.batch_receive_messages?(queue)
|
|
31
32
|
limit = batch ? FETCH_LIMIT : available_processors
|
|
32
33
|
|
|
33
34
|
if (sqs_msgs = Array(receive_messages(queue, limit))).any?
|
|
34
|
-
logger.info "Found #{sqs_msgs.size} messages for '#{queue}'"
|
|
35
|
+
logger.info { "Found #{sqs_msgs.size} messages for '#{queue}'" }
|
|
35
36
|
|
|
36
37
|
if batch
|
|
37
38
|
@manager.async.assign(queue, patch_sqs_msgs!(sqs_msgs))
|
|
@@ -41,7 +42,7 @@ module Shoryuken
|
|
|
41
42
|
|
|
42
43
|
@manager.async.rebalance_queue_weight!(queue)
|
|
43
44
|
else
|
|
44
|
-
logger.debug "No message found for '#{queue}'"
|
|
45
|
+
logger.debug { "No message found for '#{queue}'" }
|
|
45
46
|
|
|
46
47
|
@manager.async.pause_queue!(queue)
|
|
47
48
|
end
|
|
@@ -50,14 +51,15 @@ module Shoryuken
|
|
|
50
51
|
|
|
51
52
|
logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
|
|
52
53
|
rescue => ex
|
|
53
|
-
logger.error "Error fetching message: #{ex}"
|
|
54
|
-
logger.error ex.backtrace.first
|
|
54
|
+
logger.error { "Error fetching message: #{ex}" }
|
|
55
|
+
logger.error { ex.backtrace.first }
|
|
55
56
|
|
|
56
57
|
@manager.async.dispatch
|
|
57
58
|
end
|
|
58
59
|
end
|
|
59
60
|
|
|
60
61
|
end
|
|
62
|
+
|
|
61
63
|
private
|
|
62
64
|
|
|
63
65
|
def patch_sqs_msgs!(sqs_msgs)
|
data/lib/shoryuken/launcher.rb
CHANGED
|
@@ -36,7 +36,7 @@ module Shoryuken
|
|
|
36
36
|
|
|
37
37
|
def actor_died(actor, reason)
|
|
38
38
|
return if @done
|
|
39
|
-
logger.warn 'Shoryuken died due to the following error, cannot recover, process exiting'
|
|
39
|
+
logger.warn { 'Shoryuken died due to the following error, cannot recover, process exiting' }
|
|
40
40
|
exit 1
|
|
41
41
|
end
|
|
42
42
|
end
|
data/lib/shoryuken/logging.rb
CHANGED
|
@@ -17,12 +17,10 @@ module Shoryuken
|
|
|
17
17
|
end
|
|
18
18
|
|
|
19
19
|
def self.with_context(msg)
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
Thread.current[:shoryuken_context] = nil
|
|
25
|
-
end
|
|
20
|
+
Thread.current[:shoryuken_context] = msg
|
|
21
|
+
yield
|
|
22
|
+
ensure
|
|
23
|
+
Thread.current[:shoryuken_context] = nil
|
|
26
24
|
end
|
|
27
25
|
|
|
28
26
|
def self.initialize_logger(log_target = STDOUT)
|
data/lib/shoryuken/manager.rb
CHANGED
|
@@ -23,7 +23,7 @@ module Shoryuken
|
|
|
23
23
|
end
|
|
24
24
|
|
|
25
25
|
def start
|
|
26
|
-
logger.info 'Starting'
|
|
26
|
+
logger.info { 'Starting' }
|
|
27
27
|
|
|
28
28
|
dispatch
|
|
29
29
|
end
|
|
@@ -32,8 +32,8 @@ module Shoryuken
|
|
|
32
32
|
watchdog('Manager#stop died') do
|
|
33
33
|
@done = true
|
|
34
34
|
|
|
35
|
-
if callback = Shoryuken.stop_callback
|
|
36
|
-
logger.info
|
|
35
|
+
if (callback = Shoryuken.stop_callback)
|
|
36
|
+
logger.info { 'Calling Shoryuken.on_stop block' }
|
|
37
37
|
callback.call
|
|
38
38
|
end
|
|
39
39
|
|
|
@@ -58,7 +58,7 @@ module Shoryuken
|
|
|
58
58
|
|
|
59
59
|
def processor_done(queue, processor)
|
|
60
60
|
watchdog('Manager#processor_done died') do
|
|
61
|
-
logger.
|
|
61
|
+
logger.debug { "Process done for '#{queue}'" }
|
|
62
62
|
|
|
63
63
|
@threads.delete(processor.object_id)
|
|
64
64
|
@busy.delete processor
|
|
@@ -73,7 +73,7 @@ module Shoryuken
|
|
|
73
73
|
|
|
74
74
|
def processor_died(processor, reason)
|
|
75
75
|
watchdog("Manager#processor_died died") do
|
|
76
|
-
logger.error "Process died, reason: #{reason}" unless reason.to_s.empty?
|
|
76
|
+
logger.error { "Process died, reason: #{reason}" unless reason.to_s.empty? }
|
|
77
77
|
|
|
78
78
|
@threads.delete(processor.object_id)
|
|
79
79
|
@busy.delete processor
|
|
@@ -89,8 +89,8 @@ module Shoryuken
|
|
|
89
89
|
end
|
|
90
90
|
|
|
91
91
|
def assign(queue, sqs_msg)
|
|
92
|
-
watchdog(
|
|
93
|
-
logger.
|
|
92
|
+
watchdog('Manager#assign died') do
|
|
93
|
+
logger.debug { "Assigning #{sqs_msg.message_id}" }
|
|
94
94
|
|
|
95
95
|
processor = @ready.pop
|
|
96
96
|
@busy << processor
|
|
@@ -102,7 +102,7 @@ module Shoryuken
|
|
|
102
102
|
def rebalance_queue_weight!(queue)
|
|
103
103
|
watchdog('Manager#rebalance_queue_weight! died') do
|
|
104
104
|
if (original = original_queue_weight(queue)) > (current = current_queue_weight(queue))
|
|
105
|
-
logger.info "Increasing '#{queue}' weight to #{current + 1}, max: #{original}"
|
|
105
|
+
logger.info { "Increasing '#{queue}' weight to #{current + 1}, max: #{original}" }
|
|
106
106
|
|
|
107
107
|
@queues << queue
|
|
108
108
|
end
|
|
@@ -112,7 +112,7 @@ module Shoryuken
|
|
|
112
112
|
def pause_queue!(queue)
|
|
113
113
|
return if !@queues.include?(queue) || Shoryuken.options[:delay].to_f <= 0
|
|
114
114
|
|
|
115
|
-
logger.debug "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because it's empty"
|
|
115
|
+
logger.debug { "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because it's empty" }
|
|
116
116
|
|
|
117
117
|
@queues.delete(queue)
|
|
118
118
|
|
|
@@ -133,7 +133,7 @@ module Shoryuken
|
|
|
133
133
|
return
|
|
134
134
|
end
|
|
135
135
|
|
|
136
|
-
if queue = next_queue
|
|
136
|
+
if (queue = next_queue)
|
|
137
137
|
@fetcher.async.fetch(queue, @ready.size)
|
|
138
138
|
else
|
|
139
139
|
logger.debug { 'Pausing fetcher, because all queues are paused' }
|
|
@@ -158,7 +158,7 @@ module Shoryuken
|
|
|
158
158
|
return if stopped?
|
|
159
159
|
|
|
160
160
|
unless @queues.include? queue
|
|
161
|
-
logger.debug "Restarting '#{queue}'"
|
|
161
|
+
logger.debug { "Restarting '#{queue}'" }
|
|
162
162
|
|
|
163
163
|
@queues << queue
|
|
164
164
|
|
|
@@ -192,7 +192,7 @@ module Shoryuken
|
|
|
192
192
|
|
|
193
193
|
unless defined?(::ActiveJob) || !Shoryuken.worker_registry.workers(queue).empty?
|
|
194
194
|
# when no worker registered pause the queue to avoid endless recursion
|
|
195
|
-
logger.debug "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because no workers registered"
|
|
195
|
+
logger.debug { "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because no workers registered" }
|
|
196
196
|
|
|
197
197
|
after(Shoryuken.options[:delay].to_f) { async.restart_queue!(queue) }
|
|
198
198
|
|
|
@@ -220,7 +220,7 @@ module Shoryuken
|
|
|
220
220
|
logger.info { "Pausing up to #{delay} seconds to allow workers to finish..." }
|
|
221
221
|
|
|
222
222
|
after(delay) do
|
|
223
|
-
watchdog(
|
|
223
|
+
watchdog('Manager#hard_shutdown_in died') do
|
|
224
224
|
if @busy.size > 0
|
|
225
225
|
logger.info { "Hard shutting down #{@busy.size} busy workers" }
|
|
226
226
|
|