shoryuken 1.0.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/.hound.yml +6 -2
  4. data/.rubocop.yml +50 -0
  5. data/CHANGELOG.md +11 -0
  6. data/Gemfile +4 -1
  7. data/README.md +6 -52
  8. data/examples/bootstrap_queues.rb +1 -1
  9. data/lib/shoryuken/cli.rb +14 -14
  10. data/lib/shoryuken/client.rb +4 -7
  11. data/lib/shoryuken/default_worker_registry.rb +1 -1
  12. data/lib/shoryuken/environment_loader.rb +34 -16
  13. data/lib/shoryuken/extensions/active_job_adapter.rb +0 -1
  14. data/lib/shoryuken/fetcher.rb +8 -6
  15. data/lib/shoryuken/launcher.rb +1 -1
  16. data/lib/shoryuken/logging.rb +4 -6
  17. data/lib/shoryuken/manager.rb +13 -13
  18. data/lib/shoryuken/message.rb +70 -0
  19. data/lib/shoryuken/middleware/chain.rb +2 -2
  20. data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +50 -0
  21. data/lib/shoryuken/middleware/server/timing.rb +4 -4
  22. data/lib/shoryuken/processor.rb +28 -14
  23. data/lib/shoryuken/queue.rb +84 -0
  24. data/lib/shoryuken/sns_arn.rb +3 -3
  25. data/lib/shoryuken/util.rb +5 -6
  26. data/lib/shoryuken/version.rb +1 -1
  27. data/lib/shoryuken/worker.rb +1 -2
  28. data/lib/shoryuken.rb +29 -7
  29. data/shoryuken.gemspec +15 -16
  30. data/spec/shoryuken/client_spec.rb +45 -4
  31. data/spec/shoryuken/default_worker_registry_spec.rb +1 -1
  32. data/spec/shoryuken/fetcher_spec.rb +6 -6
  33. data/spec/shoryuken/middleware/chain_spec.rb +3 -1
  34. data/spec/shoryuken/middleware/server/auto_delete_spec.rb +2 -2
  35. data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +77 -0
  36. data/spec/shoryuken/middleware/server/timing_spec.rb +23 -9
  37. data/spec/shoryuken/processor_spec.rb +39 -13
  38. data/spec/shoryuken/queue_spec.rb +100 -0
  39. data/spec/shoryuken/util_spec.rb +1 -1
  40. data/spec/shoryuken/worker_spec.rb +0 -13
  41. data/spec/shoryuken_endpoint.yml +6 -0
  42. data/spec/spec_helper.rb +4 -4
  43. metadata +36 -39
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9072ce45c134cd9563ee7dddac27bc7d19733cde
4
- data.tar.gz: 2b48cb3b0d594ef1111456d0f9101ce154056995
3
+ metadata.gz: ee01f5dc2129537f640aba700e43c8be0f37e77c
4
+ data.tar.gz: 54c1fe1a997a29a77939336ca5f8f2c7e691a745
5
5
  SHA512:
6
- metadata.gz: 62bde57b564da861f90c2a68816a69743766f1120dd146817f884b2d4d2e5e5119dcef8858516b7d2581517a04177ec0edb151a97f254c6185ae9092c96efab2
7
- data.tar.gz: e011b7d261eaf6e4b87d7dc322d2f458da09fdcd5074eac90eedb8101c082b2987f303b557ba20d90c89b321a7d4df7d613852890b6888263ccbc8adb1bf77db
6
+ metadata.gz: 5543a21bcfadc457a3bb7444b27ba97d59c9f238bbc9633a100555154823132f7ee6f3cf233f84743ed553cc69a27f8ddf836515690a120b291ffb8212db8783
7
+ data.tar.gz: bcdc762ee9e02366ad4194e372ad76e2e7c23f7e5f15129d8c90d6abfa65af337c4900addaf47d14f8f6578e084a6fd84dad478878589e0c3e2912c5851c89f5
data/.gitignore CHANGED
@@ -24,3 +24,4 @@ shoryuken.yml
24
24
  *.pid
25
25
  *.log
26
26
  .env
27
+ rubocop.html
data/.hound.yml CHANGED
@@ -1,2 +1,6 @@
1
- StringLiterals:
2
- EnforcedStyle: single_quotes
1
+ ruby:
2
+ enabled: true
3
+ config_file: .rubocop.yml
4
+
5
+ java_script:
6
+ enabled: true
data/.rubocop.yml ADDED
@@ -0,0 +1,50 @@
1
+ LineLength:
2
+ Max: 120
3
+
4
+ Style/SignalException:
5
+ Enabled: false
6
+
7
+ Style/SpaceAroundEqualsInParameterDefault:
8
+ Enabled: false
9
+
10
+ Style/Documentation:
11
+ Enabled: false
12
+
13
+ Style/ClassAndModuleChildren:
14
+ Enabled: false
15
+
16
+ Metrics/PerceivedComplexity:
17
+ Enabled: false
18
+
19
+ Metrics/CyclomaticComplexity:
20
+ Enabled: false
21
+
22
+ Style/CommentAnnotation:
23
+ Enabled: false
24
+
25
+ Metrics/ClassLength:
26
+ Enabled: false
27
+
28
+ Metrics/ParameterLists:
29
+ Enabled: false
30
+
31
+ Metrics/MethodLength :
32
+ Enabled: false
33
+
34
+ Style/PerlBackrefs:
35
+ Enabled: false
36
+
37
+ Style/StringLiterals:
38
+ EnforcedStyle: single_quotes
39
+ SupportedStyles:
40
+ - single_quotes
41
+ - double_quotes
42
+
43
+ Style/StringLiteralsInInterpolation:
44
+ EnforcedStyle: single_quotes
45
+ SupportedStyles:
46
+ - single_quotes
47
+ - double_quotes
48
+
49
+ Lint/UnusedMethodArgument:
50
+ Enabled: false
data/CHANGELOG.md ADDED
@@ -0,0 +1,11 @@
1
+ ## [v2.0.0] - 2015-09-22
2
+
3
+ - Allow configuration of SQS/SNS endpoints via environment variables
4
+ - [#130](https://github.com/phstc/shoryuken/pull/130)
5
+
6
+ - Expose queue_name in the message object
7
+ - [#127](https://github.com/phstc/shoryuken/pull/127)
8
+
9
+ - README updates
10
+ - [#122](https://github.com/phstc/shoryuken/pull/122)
11
+ - [#120](https://github.com/phstc/shoryuken/pull/120)
data/Gemfile CHANGED
@@ -3,4 +3,7 @@ source 'https://rubygems.org'
3
3
  # Specify your gem's dependencies in shoryuken.gemspec
4
4
  gemspec
5
5
 
6
- gem 'codeclimate-test-reporter', group: :test, require: nil
6
+ group :test do
7
+ gem 'codeclimate-test-reporter', require: nil
8
+ gem 'multi_xml'
9
+ end
data/README.md CHANGED
@@ -36,7 +36,7 @@ If all queues get empty, all processors will be changed to the waiting state and
36
36
 
37
37
  ### Fetch in batches
38
38
 
39
- To be even more performance and cost efficient, Shoryuken fetches SQS messages in batches, so a single SQS request can fetch up to 10 messages.
39
+ To be even more performant and cost effective, Shoryuken fetches SQS messages in batches, so a single SQS request can fetch up to 10 messages.
40
40
 
41
41
  ## Requirements
42
42
 
@@ -114,7 +114,7 @@ aws:
114
114
  access_key_id: ... # or <%= ENV['AWS_ACCESS_KEY_ID'] %>
115
115
  secret_access_key: ... # or <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
116
116
  region: us-east-1 # or <%= ENV['AWS_REGION'] %>
117
- receive_message: # See http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Queue.html#receive_messages-instance_method
117
+ receive_message: # See http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#receive_message-instance_method
118
118
  # wait_time_seconds: N # The number of seconds to wait for new messages when polling. Defaults to the #wait_time_seconds defined on the queue
119
119
  attribute_names:
120
120
  - ApproximateReceiveCount
@@ -132,59 +132,13 @@ The ```aws``` section is used to configure both the Aws objects used by Shoryuke
132
132
  - ```account_id``` is used when generating SNS ARNs
133
133
  - ```sns_endpoint``` can be used to explicitly override the SNS endpoint
134
134
  - ```sqs_endpoint``` can be used to explicitly override the SQS endpoint
135
- - ```receive_message``` can be used to define the options passed to the http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Queue.html#receive_messages-instance_method
135
+ - ```receive_message``` can be used to define the options passed to the http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS/Client.html#receive_message-instance_method
136
136
 
137
- ### Rails Integration
138
-
139
- You can tell Shoryuken to load your Rails application by passing the `-R` or `--rails` flag to the "shoryuken" command.
140
-
141
- If you load Rails, and assuming your workers are located in the `app/workers` directory, they will be auto-loaded. This means you don't need to require them explicitly with `-r`.
142
-
143
- For middleware and other configuration, you might want to create an initializer:
144
-
145
- ```ruby
146
- Shoryuken.configure_server do |config|
147
- # Replace Rails logger so messages are logged wherever Shoryuken is logging
148
- # Note: this entire block is only run by the processor, so we don't overwrite
149
- # the logger when the app is running as usual.
150
- Rails.logger = Shoryuken::Logging.logger
151
-
152
- config.server_middleware do |chain|
153
- chain.add Shoryuken::MyMiddleware
154
- end
155
- end
156
- ```
157
-
158
- *Note:* In the above case, since we are replacing the Rails logger, it's desired that this initializer runs before other initializers (in case they themselves use the logger). Since by Rails conventions initializers are executed in alphabetical order, this can be achieved by prepending the initializer filename with `00_` (assuming no other initializers alphabetically precede this one).
159
-
160
- This feature works for Rails 4+, but needs to be confirmed for older versions.
161
-
162
- #### ActiveJob Support
137
+ The ```sns_endpoint``` and ```sqs_endpoint``` Shoryuken-specific options will also fallback to the environment variables ```AWS_SNS_ENDPOINT``` and ```AWS_SQS_ENDPOINT``` respectively, if they are set.
163
138
 
164
- Yes, Shoryuken supports ActiveJob! This means that you can put your jobs in processor-agnostic `ActiveJob::Base` subclasses, and change processors whenever you want (or better yet, switch to Shoryuken from another processor easily!).
165
-
166
- It works as expected. Just put your job in `app/jobs`. Here's an example:
167
-
168
- ```ruby
169
- class ProcessPhotoJob < ActiveJob::Base
170
- queue_as :default
171
-
172
- rescue_from ActiveJob::DeserializationError do |e|
173
- Shoryuken.logger.error ex
174
- Shoryuken.logger.error ex.backtrace.join("\n")
175
- end
176
-
177
- def perform(photo)
178
- photo.process_image!
179
- end
180
- end
181
- ```
182
-
183
- Delayed mailers, ActiveRecord serialization, etc. all work.
184
-
185
- See [ActiveJob docs](http://edgeguides.rubyonrails.org/active_job_basics.html) for more info.
139
+ ### Rails Integration
186
140
 
187
- *Note:* When queueing jobs to be performed in the future (e.g when setting the `wait` or `wait_until` ActiveJob options), SQS limits the amount of time to 15 minutes. Shoryuken will raise an exception if you attempt to schedule a job further into the future than this limit.
141
+ [Check the Rails Integration Active Job documention](https://github.com/phstc/shoryuken/wiki/Rails-Integration-Active-Job).
188
142
 
189
143
  ### Start Shoryuken
190
144
 
@@ -22,7 +22,7 @@ if sqs.config['endpoint'] =~ /amazonaws.com/
22
22
  ).attributes['QueueArn']
23
23
 
24
24
  attributes = {}
25
- attributes['RedrivePolicy'] = %Q{{"maxReceiveCount":"7", "deadLetterTargetArn":"#{dead_letter_queue_arn}"}}
25
+ attributes['RedrivePolicy'] = %Q({"maxReceiveCount":"7", "deadLetterTargetArn":"#{dead_letter_queue_arn}"})
26
26
 
27
27
  sqs.set_queue_attributes queue_url: default_queue_url, attributes: attributes
28
28
  end
data/lib/shoryuken/cli.rb CHANGED
@@ -19,7 +19,7 @@ module Shoryuken
19
19
  def run(args)
20
20
  self_read, self_write = IO.pipe
21
21
 
22
- %w[INT TERM USR1 USR2 TTIN].each do |sig|
22
+ %w(INT TERM USR1 USR2 TTIN).each do |sig|
23
23
  begin
24
24
  trap sig do
25
25
  self_write.puts(sig)
@@ -41,14 +41,14 @@ module Shoryuken
41
41
  @launcher = Shoryuken::Launcher.new
42
42
 
43
43
  if callback = Shoryuken.start_callback
44
- logger.info "Calling Shoryuken.on_start block"
44
+ logger.info { 'Calling Shoryuken.on_start block' }
45
45
  callback.call
46
46
  end
47
47
 
48
48
  begin
49
49
  launcher.run
50
50
 
51
- while readable_io = IO.select([self_read])
51
+ while (readable_io = IO.select([self_read]))
52
52
  signal = readable_io.first[0].gets.strip
53
53
  handle_signal(signal)
54
54
  end
@@ -86,7 +86,7 @@ module Shoryuken
86
86
 
87
87
  files_to_reopen.each do |file|
88
88
  begin
89
- file.reopen file.path, "a+"
89
+ file.reopen file.path, 'a+'
90
90
  file.sync = true
91
91
  rescue ::Exception
92
92
  end
@@ -102,7 +102,7 @@ module Shoryuken
102
102
  end
103
103
 
104
104
  def write_pid
105
- if path = Shoryuken.options[:pidfile]
105
+ if (path = Shoryuken.options[:pidfile])
106
106
  File.open(path, 'w') do |f|
107
107
  f.puts Process.pid
108
108
  end
@@ -150,7 +150,7 @@ module Shoryuken
150
150
  opts[:verbose] = arg
151
151
  end
152
152
 
153
- o.on '-V', '--version', 'Print version and exit' do |arg|
153
+ o.on '-V', '--version', 'Print version and exit' do
154
154
  puts "Shoryuken #{Shoryuken::VERSION}"
155
155
  exit 0
156
156
  end
@@ -158,7 +158,7 @@ module Shoryuken
158
158
 
159
159
  @parser.banner = 'shoryuken [options]'
160
160
  @parser.on_tail '-h', '--help', 'Show help' do
161
- logger.info @parser
161
+ logger.info { @parser }
162
162
  exit 1
163
163
  end
164
164
  @parser.parse!(argv)
@@ -166,22 +166,22 @@ module Shoryuken
166
166
  end
167
167
 
168
168
  def handle_signal(sig)
169
- logger.info "Got #{sig} signal"
169
+ logger.info { "Got #{sig} signal" }
170
170
 
171
171
  case sig
172
172
  when 'USR1'
173
- logger.info "Received USR1, will soft shutdown down"
173
+ logger.info { 'Received USR1, will soft shutdown down' }
174
174
 
175
175
  launcher.stop
176
176
 
177
177
  exit 0
178
178
  when 'TTIN'
179
179
  Thread.list.each do |thread|
180
- logger.info "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
180
+ logger.info { "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}" }
181
181
  if thread.backtrace
182
- logger.info thread.backtrace.join("\n")
182
+ logger.info { thread.backtrace.join("\n") }
183
183
  else
184
- logger.info "<no backtrace available>"
184
+ logger.info { '<no backtrace available>' }
185
185
  end
186
186
  end
187
187
 
@@ -189,9 +189,9 @@ module Shoryuken
189
189
  busy = launcher.manager.instance_variable_get(:@busy).size
190
190
  queues = launcher.manager.instance_variable_get(:@queues)
191
191
 
192
- logger.info "Ready: #{ready}, Busy: #{busy}, Active Queues: #{unparse_queues(queues)}"
192
+ logger.info { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{unparse_queues(queues)}" }
193
193
  else
194
- logger.info "Received #{sig}, will shutdown down"
194
+ logger.info { "Received #{sig}, will shutdown down" }
195
195
 
196
196
  raise Interrupt
197
197
  end
@@ -5,7 +5,7 @@ module Shoryuken
5
5
 
6
6
  class << self
7
7
  def queues(name)
8
- @@queues[name.to_s] ||= sqs_resource.get_queue_by_name(queue_name: name)
8
+ @@queues[name.to_s] ||= Shoryuken::Queue.new(sqs, name)
9
9
  end
10
10
 
11
11
  def sns
@@ -20,10 +20,6 @@ module Shoryuken
20
20
  @sqs ||= Aws::SQS::Client.new(aws_client_options(:sqs_endpoint))
21
21
  end
22
22
 
23
- def sqs_resource
24
- @sqs_resource ||= Aws::SQS::Resource.new(client: sqs)
25
- end
26
-
27
23
  def topics(name)
28
24
  @@topics[name.to_s] ||= Topic.new(name, sns)
29
25
  end
@@ -33,8 +29,9 @@ module Shoryuken
33
29
 
34
30
  private
35
31
 
36
- def aws_client_options service_endpoint_key
37
- explicit_endpoint = Shoryuken.options[:aws][service_endpoint_key]
32
+ def aws_client_options(service_endpoint_key)
33
+ environment_endpoint = ENV["AWS_#{service_endpoint_key.to_s.upcase}"]
34
+ explicit_endpoint = Shoryuken.options[:aws][service_endpoint_key] || environment_endpoint
38
35
  options = {}
39
36
  options[:endpoint] = explicit_endpoint unless explicit_endpoint.to_s.empty?
40
37
  options
@@ -28,7 +28,7 @@ module Shoryuken
28
28
  end
29
29
 
30
30
  def register_worker(queue, clazz)
31
- if worker_class = @workers[queue]
31
+ if (worker_class = @workers[queue])
32
32
  if worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true
33
33
  raise ArgumentError, "Could not register #{clazz} for '#{queue}', "\
34
34
  "because #{worker_class} is already registered for this queue, "\
@@ -10,7 +10,7 @@ module Shoryuken
10
10
  load(config_file: (Rails.root + 'config' + 'shoryuken.yml'))
11
11
  end
12
12
 
13
- def initialize options
13
+ def initialize(options)
14
14
  @options = options
15
15
  end
16
16
 
@@ -19,6 +19,7 @@ module Shoryuken
19
19
  load_rails if options[:rails]
20
20
  Shoryuken.options.merge!(config_file_options)
21
21
  merge_cli_defined_queues
22
+ prefix_active_job_queue_names
22
23
  Shoryuken.options.merge!(options)
23
24
  parse_queues
24
25
  require_workers
@@ -33,7 +34,7 @@ module Shoryuken
33
34
  def config_file_options
34
35
  if (path = options[:config_file])
35
36
  unless File.exist?(path)
36
- Shoryuken.logger.warn "Config file #{path} does not exist"
37
+ Shoryuken.logger.warn { "Config file #{path} does not exist" }
37
38
  path = nil
38
39
  end
39
40
  end
@@ -48,24 +49,26 @@ module Shoryuken
48
49
  # when not explicit supplied
49
50
  return if Shoryuken.options[:aws].empty?
50
51
 
51
- shoryuken_keys = %i(
52
+ shoryuken_keys = %w(
52
53
  account_id
53
54
  sns_endpoint
54
55
  sqs_endpoint
55
- receive_message)
56
+ receive_message).map(&:to_sym)
56
57
 
57
58
  aws_options = Shoryuken.options[:aws].reject do |k, v|
58
59
  shoryuken_keys.include?(k)
59
60
  end
60
61
 
62
+ # assume credentials based authentication
61
63
  credentials = Aws::Credentials.new(
62
64
  aws_options.delete(:access_key_id),
63
65
  aws_options.delete(:secret_access_key))
64
66
 
65
- aws_options = aws_options.merge(credentials: credentials)
67
+ # but only if the configuration options have valid values
68
+ aws_options = aws_options.merge(credentials: credentials) if credentials.set?
66
69
 
67
- if callback = Shoryuken.aws_initialization_callback
68
- Shoryuken.logger.info "Calling Shoryuken.on_aws_initialization block"
70
+ if (callback = Shoryuken.aws_initialization_callback)
71
+ Shoryuken.logger.info { 'Calling Shoryuken.on_aws_initialization block' }
69
72
  callback.call(aws_options)
70
73
  end
71
74
 
@@ -82,19 +85,19 @@ module Shoryuken
82
85
 
83
86
  require 'rails'
84
87
  if ::Rails::VERSION::MAJOR < 4
85
- require File.expand_path("config/environment.rb")
88
+ require File.expand_path('config/environment.rb')
86
89
  ::Rails.application.eager_load!
87
90
  else
88
91
  # Painful contortions, see 1791 for discussion
89
- require File.expand_path("config/application.rb")
90
- ::Rails::Application.initializer "shoryuken.eager_load" do
92
+ require File.expand_path('config/application.rb')
93
+ ::Rails::Application.initializer 'shoryuken.eager_load' do
91
94
  ::Rails.application.config.eager_load = true
92
95
  end
93
96
  require 'shoryuken/extensions/active_job_adapter' if defined?(::ActiveJob)
94
- require File.expand_path("config/environment.rb")
97
+ require File.expand_path('config/environment.rb')
95
98
  end
96
99
 
97
- Shoryuken.logger.info "Rails environment loaded"
100
+ Shoryuken.logger.info { 'Rails environment loaded' }
98
101
  end
99
102
 
100
103
  def merge_cli_defined_queues
@@ -109,6 +112,21 @@ module Shoryuken
109
112
  end
110
113
  end
111
114
 
115
+ def prefix_active_job_queue_names
116
+ return unless @options[:rails]
117
+ return unless Shoryuken.active_job_queue_name_prefixing
118
+
119
+ queue_name_prefix = ::ActiveJob::Base.queue_name_prefix
120
+ queue_name_delimiter = ::ActiveJob::Base.queue_name_delimiter
121
+
122
+ # See https://github.com/rails/rails/blob/master/activejob/lib/active_job/queue_name.rb#L27
123
+ Shoryuken.options[:queues].to_a.map! do |queue_name, weight|
124
+ name_parts = [queue_name_prefix.presence, queue_name]
125
+ prefixed_queue_name = name_parts.compact.join(queue_name_delimiter)
126
+ [prefixed_queue_name, weight]
127
+ end
128
+ end
129
+
112
130
  def parse_queue(queue, weight = nil)
113
131
  [weight.to_i, 1].max.times { Shoryuken.queues << queue }
114
132
  end
@@ -123,7 +141,7 @@ module Shoryuken
123
141
  Shoryuken.worker_registry.queues.each do |queue|
124
142
  Shoryuken.worker_registry.workers(queue).each do |worker_class|
125
143
  if worker_class.instance_method(:perform).arity == 1
126
- Shoryuken.logger.warn "[DEPRECATION] #{worker_class.name}#perform(sqs_msg) is deprecated. Please use #{worker_class.name}#perform(sqs_msg, body)"
144
+ Shoryuken.logger.warn { "[DEPRECATION] #{worker_class.name}#perform(sqs_msg) is deprecated. Please use #{worker_class.name}#perform(sqs_msg, body)" }
127
145
 
128
146
  worker_class.class_eval do
129
147
  alias_method :deprecated_perform, :perform
@@ -142,13 +160,13 @@ module Shoryuken
142
160
  end
143
161
 
144
162
  def validate_queues
145
- Shoryuken.logger.warn 'No queues supplied' if Shoryuken.queues.empty?
163
+ Shoryuken.logger.warn { 'No queues supplied' } if Shoryuken.queues.empty?
146
164
 
147
165
  Shoryuken.queues.uniq.each do |queue|
148
166
  begin
149
167
  Shoryuken::Client.queues queue
150
168
  rescue Aws::SQS::Errors::NonExistentQueue
151
- Shoryuken.logger.warn "AWS Queue '#{queue}' does not exist"
169
+ Shoryuken.logger.warn { "AWS Queue '#{queue}' does not exist" }
152
170
  end
153
171
  end
154
172
  end
@@ -159,7 +177,7 @@ module Shoryuken
159
177
 
160
178
  unless defined?(::ActiveJob)
161
179
  (all_queues - queues_with_workers).each do |queue|
162
- Shoryuken.logger.warn "No worker supplied for '#{queue}'"
180
+ Shoryuken.logger.warn { "No worker supplied for '#{queue}'" }
163
181
  end
164
182
  end
165
183
  end
@@ -37,7 +37,6 @@ module ActiveJob
37
37
 
38
38
  def message(job, options = {})
39
39
  body = job.serialize
40
- body = JSON.dump(body) if body.is_a?(Hash)
41
40
 
42
41
  { message_body: body,
43
42
  message_attributes: message_attributes }.merge(options)
@@ -13,9 +13,10 @@ module Shoryuken
13
13
  # AWS limits the batch size by 10
14
14
  limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
15
15
 
16
- options = Shoryuken.options[:aws][:receive_message].to_h.dup
16
+ options = (Shoryuken.options[:aws][:receive_message] || {}).dup
17
17
  options[:max_number_of_messages] = limit
18
18
  options[:message_attribute_names] = %w(All)
19
+ options[:attribute_names] = %w(All)
19
20
 
20
21
  Shoryuken::Client.queues(queue).receive_messages options
21
22
  end
@@ -24,14 +25,14 @@ module Shoryuken
24
25
  watchdog('Fetcher#fetch died') do
25
26
  started_at = Time.now
26
27
 
27
- logger.debug "Looking for new messages in '#{queue}'"
28
+ logger.debug { "Looking for new messages in '#{queue}'" }
28
29
 
29
30
  begin
30
31
  batch = Shoryuken.worker_registry.batch_receive_messages?(queue)
31
32
  limit = batch ? FETCH_LIMIT : available_processors
32
33
 
33
34
  if (sqs_msgs = Array(receive_messages(queue, limit))).any?
34
- logger.info "Found #{sqs_msgs.size} messages for '#{queue}'"
35
+ logger.info { "Found #{sqs_msgs.size} messages for '#{queue}'" }
35
36
 
36
37
  if batch
37
38
  @manager.async.assign(queue, patch_sqs_msgs!(sqs_msgs))
@@ -41,7 +42,7 @@ module Shoryuken
41
42
 
42
43
  @manager.async.rebalance_queue_weight!(queue)
43
44
  else
44
- logger.debug "No message found for '#{queue}'"
45
+ logger.debug { "No message found for '#{queue}'" }
45
46
 
46
47
  @manager.async.pause_queue!(queue)
47
48
  end
@@ -50,14 +51,15 @@ module Shoryuken
50
51
 
51
52
  logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
52
53
  rescue => ex
53
- logger.error "Error fetching message: #{ex}"
54
- logger.error ex.backtrace.first
54
+ logger.error { "Error fetching message: #{ex}" }
55
+ logger.error { ex.backtrace.first }
55
56
 
56
57
  @manager.async.dispatch
57
58
  end
58
59
  end
59
60
 
60
61
  end
62
+
61
63
  private
62
64
 
63
65
  def patch_sqs_msgs!(sqs_msgs)
@@ -36,7 +36,7 @@ module Shoryuken
36
36
 
37
37
  def actor_died(actor, reason)
38
38
  return if @done
39
- logger.warn 'Shoryuken died due to the following error, cannot recover, process exiting'
39
+ logger.warn { 'Shoryuken died due to the following error, cannot recover, process exiting' }
40
40
  exit 1
41
41
  end
42
42
  end
@@ -17,12 +17,10 @@ module Shoryuken
17
17
  end
18
18
 
19
19
  def self.with_context(msg)
20
- begin
21
- Thread.current[:shoryuken_context] = msg
22
- yield
23
- ensure
24
- Thread.current[:shoryuken_context] = nil
25
- end
20
+ Thread.current[:shoryuken_context] = msg
21
+ yield
22
+ ensure
23
+ Thread.current[:shoryuken_context] = nil
26
24
  end
27
25
 
28
26
  def self.initialize_logger(log_target = STDOUT)
@@ -23,7 +23,7 @@ module Shoryuken
23
23
  end
24
24
 
25
25
  def start
26
- logger.info 'Starting'
26
+ logger.info { 'Starting' }
27
27
 
28
28
  dispatch
29
29
  end
@@ -32,8 +32,8 @@ module Shoryuken
32
32
  watchdog('Manager#stop died') do
33
33
  @done = true
34
34
 
35
- if callback = Shoryuken.stop_callback
36
- logger.info "Calling Shoryuken.on_stop block"
35
+ if (callback = Shoryuken.stop_callback)
36
+ logger.info { 'Calling Shoryuken.on_stop block' }
37
37
  callback.call
38
38
  end
39
39
 
@@ -58,7 +58,7 @@ module Shoryuken
58
58
 
59
59
  def processor_done(queue, processor)
60
60
  watchdog('Manager#processor_done died') do
61
- logger.info "Process done for '#{queue}'"
61
+ logger.debug { "Process done for '#{queue}'" }
62
62
 
63
63
  @threads.delete(processor.object_id)
64
64
  @busy.delete processor
@@ -73,7 +73,7 @@ module Shoryuken
73
73
 
74
74
  def processor_died(processor, reason)
75
75
  watchdog("Manager#processor_died died") do
76
- logger.error "Process died, reason: #{reason}" unless reason.to_s.empty?
76
+ logger.error { "Process died, reason: #{reason}" unless reason.to_s.empty? }
77
77
 
78
78
  @threads.delete(processor.object_id)
79
79
  @busy.delete processor
@@ -89,8 +89,8 @@ module Shoryuken
89
89
  end
90
90
 
91
91
  def assign(queue, sqs_msg)
92
- watchdog("Manager#assign died") do
93
- logger.info "Assigning #{sqs_msg.message_id}"
92
+ watchdog('Manager#assign died') do
93
+ logger.debug { "Assigning #{sqs_msg.message_id}" }
94
94
 
95
95
  processor = @ready.pop
96
96
  @busy << processor
@@ -102,7 +102,7 @@ module Shoryuken
102
102
  def rebalance_queue_weight!(queue)
103
103
  watchdog('Manager#rebalance_queue_weight! died') do
104
104
  if (original = original_queue_weight(queue)) > (current = current_queue_weight(queue))
105
- logger.info "Increasing '#{queue}' weight to #{current + 1}, max: #{original}"
105
+ logger.info { "Increasing '#{queue}' weight to #{current + 1}, max: #{original}" }
106
106
 
107
107
  @queues << queue
108
108
  end
@@ -112,7 +112,7 @@ module Shoryuken
112
112
  def pause_queue!(queue)
113
113
  return if !@queues.include?(queue) || Shoryuken.options[:delay].to_f <= 0
114
114
 
115
- logger.debug "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because it's empty"
115
+ logger.debug { "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because it's empty" }
116
116
 
117
117
  @queues.delete(queue)
118
118
 
@@ -133,7 +133,7 @@ module Shoryuken
133
133
  return
134
134
  end
135
135
 
136
- if queue = next_queue
136
+ if (queue = next_queue)
137
137
  @fetcher.async.fetch(queue, @ready.size)
138
138
  else
139
139
  logger.debug { 'Pausing fetcher, because all queues are paused' }
@@ -158,7 +158,7 @@ module Shoryuken
158
158
  return if stopped?
159
159
 
160
160
  unless @queues.include? queue
161
- logger.debug "Restarting '#{queue}'"
161
+ logger.debug { "Restarting '#{queue}'" }
162
162
 
163
163
  @queues << queue
164
164
 
@@ -192,7 +192,7 @@ module Shoryuken
192
192
 
193
193
  unless defined?(::ActiveJob) || !Shoryuken.worker_registry.workers(queue).empty?
194
194
  # when no worker registered pause the queue to avoid endless recursion
195
- logger.debug "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because no workers registered"
195
+ logger.debug { "Pausing '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because no workers registered" }
196
196
 
197
197
  after(Shoryuken.options[:delay].to_f) { async.restart_queue!(queue) }
198
198
 
@@ -220,7 +220,7 @@ module Shoryuken
220
220
  logger.info { "Pausing up to #{delay} seconds to allow workers to finish..." }
221
221
 
222
222
  after(delay) do
223
- watchdog("Manager#hard_shutdown_in died") do
223
+ watchdog('Manager#hard_shutdown_in died') do
224
224
  if @busy.size > 0
225
225
  logger.info { "Hard shutting down #{@busy.size} busy workers" }
226
226