shoryuken 2.1.3 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/.codeclimate.yml +2 -0
  3. data/.rubocop.yml +8 -2
  4. data/.travis.yml +1 -0
  5. data/CHANGELOG.md +19 -0
  6. data/README.md +20 -104
  7. data/Rakefile +0 -1
  8. data/bin/cli/base.rb +42 -0
  9. data/bin/cli/sqs.rb +188 -0
  10. data/bin/shoryuken +47 -9
  11. data/examples/default_worker.rb +1 -1
  12. data/lib/shoryuken.rb +75 -55
  13. data/lib/shoryuken/client.rb +3 -15
  14. data/lib/shoryuken/default_worker_registry.rb +9 -5
  15. data/lib/shoryuken/environment_loader.rb +9 -40
  16. data/lib/shoryuken/fetcher.rb +16 -18
  17. data/lib/shoryuken/launcher.rb +5 -28
  18. data/lib/shoryuken/manager.rb +60 -140
  19. data/lib/shoryuken/message.rb +4 -13
  20. data/lib/shoryuken/middleware/chain.rb +1 -18
  21. data/lib/shoryuken/middleware/server/auto_extend_visibility.rb +7 -16
  22. data/lib/shoryuken/middleware/server/exponential_backoff_retry.rb +25 -21
  23. data/lib/shoryuken/polling.rb +2 -4
  24. data/lib/shoryuken/processor.rb +2 -11
  25. data/lib/shoryuken/queue.rb +1 -3
  26. data/lib/shoryuken/runner.rb +143 -0
  27. data/lib/shoryuken/util.rb +0 -8
  28. data/lib/shoryuken/version.rb +1 -1
  29. data/lib/shoryuken/worker.rb +1 -1
  30. data/shoryuken.gemspec +6 -5
  31. data/spec/integration/launcher_spec.rb +4 -3
  32. data/spec/shoryuken/client_spec.rb +2 -45
  33. data/spec/shoryuken/default_worker_registry_spec.rb +12 -10
  34. data/spec/shoryuken/environment_loader_spec.rb +34 -0
  35. data/spec/shoryuken/manager_spec.rb +11 -21
  36. data/spec/shoryuken/middleware/chain_spec.rb +0 -24
  37. data/spec/shoryuken/middleware/server/auto_extend_visibility_spec.rb +0 -2
  38. data/spec/shoryuken/middleware/server/exponential_backoff_retry_spec.rb +46 -29
  39. data/spec/shoryuken/processor_spec.rb +5 -5
  40. data/spec/shoryuken/{cli_spec.rb → runner_spec.rb} +8 -22
  41. data/spec/shoryuken_spec.rb +13 -1
  42. data/spec/spec_helper.rb +3 -8
  43. metadata +29 -22
  44. data/lib/shoryuken/aws_config.rb +0 -64
  45. data/lib/shoryuken/cli.rb +0 -215
  46. data/lib/shoryuken/sns_arn.rb +0 -27
  47. data/lib/shoryuken/topic.rb +0 -17
  48. data/spec/shoryuken/sns_arn_spec.rb +0 -42
  49. data/spec/shoryuken/topic_spec.rb +0 -32
  50. data/spec/shoryuken_endpoint.yml +0 -6
data/bin/shoryuken CHANGED
@@ -1,12 +1,50 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
- require_relative '../lib/shoryuken/cli'
4
-
5
- begin
6
- Shoryuken::CLI.instance.run(ARGV)
7
- rescue => e
8
- raise e if $DEBUG
9
- STDERR.puts e.message
10
- STDERR.puts e.backtrace.join("\n")
11
- exit 1
3
+ require 'rubygems'
4
+
5
+ require 'thor'
6
+ require 'aws-sdk-core'
7
+ require_relative 'cli/base'
8
+ require_relative 'cli/sqs'
9
+ require_relative '../lib/shoryuken/runner'
10
+
11
+ # rubocop:disable Metrics/AbcSize
12
+ module Shoryuken
13
+ module CLI
14
+ class Runner < Base
15
+ default_task :start
16
+
17
+ register(Shoryuken::CLI::SQS, 'sqs', 'sqs COMMAND', 'SQS commands')
18
+
19
+ desc 'start', 'Start shoryuken'
20
+ method_option :concurrency, aliases: '-c', type: :numeric, desc: 'Processor threads to use'
21
+ method_option :daemon, aliases: '-d', type: :boolean, desc: 'Daemonize process'
22
+ method_option :queues, aliases: '-q', type: :array, desc: 'Queues to process with optional weights'
23
+ method_option :require, aliases: '-r', type: :string, desc: 'Dir or path of the workers'
24
+ method_option :config_file, aliases: '-C', type: :string, desc: 'Path to config file'
25
+ method_option :rails, aliases: '-R', type: :boolean, desc: 'Load Rails'
26
+ method_option :logfile, aliases: '-L', type: :string, desc: 'Path to logfile'
27
+ method_option :pidfile, aliases: '-P', type: :string, desc: 'Path to pidfile'
28
+ method_option :verbose, aliases: '-v', type: :boolean, desc: 'Print more verbose output'
29
+ def start
30
+ opts = options.to_h.symbolize_keys
31
+
32
+ # Keep compatibility with old CLI queue format
33
+ opts[:queues] = options.queues.map { |q| q.split(',') } if options.queues
34
+
35
+ if options.daemon && options.logfile.nil?
36
+ fail_task "You should set a logfile if you're going to daemonize"
37
+ end
38
+
39
+ Shoryuken::Runner.instance.run(opts.freeze)
40
+ end
41
+
42
+ desc 'version', 'Print version'
43
+ def version
44
+ say "Shoryuken #{Shoryuken::VERSION}"
45
+ end
46
+ end
47
+ end
12
48
  end
49
+
50
+ Shoryuken::CLI::Runner.start
@@ -4,6 +4,6 @@ class DefaultWorker
4
4
  shoryuken_options queue: 'default', auto_delete: true
5
5
 
6
6
  def perform(sqs_msg, body)
7
- Shoryuken.logger.info("Received message: '#{body}'")
7
+ Shoryuken.logger.debug("Received message: '#{body}'")
8
8
  end
9
9
  end
data/lib/shoryuken.rb CHANGED
@@ -1,12 +1,13 @@
1
1
  require 'yaml'
2
+ require 'json'
2
3
  require 'aws-sdk-core'
3
4
  require 'time'
5
+ require 'concurrent'
4
6
 
5
7
  require 'shoryuken/version'
6
8
  require 'shoryuken/core_ext'
7
9
  require 'shoryuken/util'
8
10
  require 'shoryuken/logging'
9
- require 'shoryuken/aws_config'
10
11
  require 'shoryuken/environment_loader'
11
12
  require 'shoryuken/queue'
12
13
  require 'shoryuken/message'
@@ -19,9 +20,11 @@ require 'shoryuken/middleware/server/auto_delete'
19
20
  Shoryuken::Middleware::Server.autoload :AutoExtendVisibility, 'shoryuken/middleware/server/auto_extend_visibility'
20
21
  require 'shoryuken/middleware/server/exponential_backoff_retry'
21
22
  require 'shoryuken/middleware/server/timing'
22
- require 'shoryuken/sns_arn'
23
- require 'shoryuken/topic'
24
23
  require 'shoryuken/polling'
24
+ require 'shoryuken/manager'
25
+ require 'shoryuken/launcher'
26
+ require 'shoryuken/processor'
27
+ require 'shoryuken/fetcher'
25
28
 
26
29
  module Shoryuken
27
30
  DEFAULTS = {
@@ -33,78 +36,106 @@ module Shoryuken
33
36
  lifecycle_events: {
34
37
  startup: [],
35
38
  quiet: [],
36
- shutdown: [],
39
+ shutdown: []
37
40
  },
38
- polling_strategy: Polling::WeightedRoundRobin,
39
- }
41
+ polling_strategy: Polling::WeightedRoundRobin
42
+ }.freeze
40
43
 
41
- @@queues = []
42
- @@worker_registry = DefaultWorkerRegistry.new
44
+ @@queues = []
45
+ @@worker_registry = DefaultWorkerRegistry.new
43
46
  @@active_job_queue_name_prefixing = false
47
+ @@sqs_client = nil
48
+ @@sqs_client_receive_message_opts = {}
49
+ @@start_callback = nil
50
+ @@stop_callback = nil
44
51
 
45
52
  class << self
46
- def options
47
- @options ||= DEFAULTS.dup
48
- end
49
-
50
53
  def queues
51
54
  @@queues
52
55
  end
53
56
 
54
- def logger
55
- Shoryuken::Logging.logger
57
+ def add_queue(queue, priority = 1)
58
+ priority.times { queues << queue }
56
59
  end
57
60
 
58
- def register_worker(*args)
59
- worker_registry.register_worker(*args)
61
+ def worker_registry
62
+ @@worker_registry
60
63
  end
61
64
 
62
65
  def worker_registry=(worker_registry)
63
66
  @@worker_registry = worker_registry
64
67
  end
65
68
 
66
- def worker_registry
67
- @@worker_registry
69
+ def start_callback
70
+ @@start_callback
71
+ end
72
+
73
+ def start_callback=(start_callback)
74
+ @@start_callback = start_callback
75
+ end
76
+
77
+ def stop_callback
78
+ @@stop_callback
79
+ end
80
+
81
+ def stop_callback=(stop_callback)
82
+ @@stop_callback = stop_callback
68
83
  end
69
84
 
70
85
  def active_job_queue_name_prefixing
71
86
  @@active_job_queue_name_prefixing
72
87
  end
73
88
 
74
- def active_job_queue_name_prefixing=(prefixing)
75
- @@active_job_queue_name_prefixing = prefixing
89
+ def active_job_queue_name_prefixing=(active_job_queue_name_prefixing)
90
+ @@active_job_queue_name_prefixing = active_job_queue_name_prefixing
91
+ end
92
+
93
+ def sqs_client
94
+ @@sqs_client ||= Aws::SQS::Client.new
95
+ end
96
+
97
+ def sqs_client=(sqs_client)
98
+ @@sqs_client
99
+ end
100
+
101
+ def sqs_client_receive_message_opts
102
+ @@sqs_client_receive_message_opts
103
+ end
104
+
105
+ def sqs_client_receive_message_opts=(sqs_client_receive_message_opts)
106
+ @@sqs_client_receive_message_opts
107
+ end
108
+
109
+ def options
110
+ @@options ||= DEFAULTS.dup
111
+ end
112
+
113
+ def logger
114
+ Shoryuken::Logging.logger
115
+ end
116
+
117
+ def register_worker(*args)
118
+ @@worker_registry.register_worker(*args)
76
119
  end
77
120
 
78
- ##
79
- # Configuration for Shoryuken server, use like:
80
- #
81
- # Shoryuken.configure_server do |config|
82
- # config.aws = { :sqs_endpoint => '...', :access_key_id: '...', :secret_access_key: '...', region: '...' }
83
- # end
84
121
  def configure_server
85
122
  yield self if server?
86
123
  end
87
124
 
88
125
  def server_middleware
89
- @server_chain ||= default_server_middleware
90
- yield @server_chain if block_given?
91
- @server_chain
126
+ @@server_chain ||= default_server_middleware
127
+ yield @@server_chain if block_given?
128
+ @@server_chain
92
129
  end
93
130
 
94
- ##
95
- # Configuration for Shoryuken client, use like:
96
- #
97
- # Shoryuken.configure_client do |config|
98
- # config.aws = { :sqs_endpoint => '...', :access_key_id: '...', :secret_access_key: '...', region: '...' }
99
- # end
100
131
  def configure_client
101
132
  yield self unless server?
102
133
  end
103
134
 
104
135
  def client_middleware
105
- @client_chain ||= default_client_middleware
106
- yield @client_chain if block_given?
107
- @client_chain
136
+ @@client_chain ||= default_client_middleware
137
+ yield @@client_chain if block_given?
138
+ @@client_chain
108
139
  end
109
140
 
110
141
  def default_worker_options
@@ -114,27 +145,20 @@ module Shoryuken
114
145
  'auto_delete' => false,
115
146
  'auto_visibility_timeout' => false,
116
147
  'retry_intervals' => nil,
117
- 'batch' => false }
148
+ 'batch' => false
149
+ }
118
150
  end
119
151
 
120
- def default_worker_options=(options)
121
- @@default_worker_options = options
122
- end
123
-
124
- def on_aws_initialization(&block)
125
- @aws_initialization_callback = block
152
+ def default_worker_options=(default_worker_options)
153
+ @@default_worker_options = default_worker_options
126
154
  end
127
155
 
128
156
  def on_start(&block)
129
- @start_callback = block
157
+ @@start_callback = block
130
158
  end
131
159
 
132
160
  def on_stop(&block)
133
- @stop_callback = block
134
- end
135
-
136
- def aws=(hash)
137
- Shoryuken::AwsConfig.setup(hash)
161
+ @@stop_callback = block
138
162
  end
139
163
 
140
164
  # Register a block to run at a point in the Shoryuken lifecycle.
@@ -151,10 +175,6 @@ module Shoryuken
151
175
  options[:lifecycle_events][event] << block
152
176
  end
153
177
 
154
- attr_reader :aws_initialization_callback,
155
- :start_callback,
156
- :stop_callback
157
-
158
178
  private
159
179
 
160
180
  def default_server_middleware
@@ -1,31 +1,19 @@
1
1
  module Shoryuken
2
2
  class Client
3
3
  @@queues = {}
4
- @@topics = {}
5
4
 
6
5
  class << self
7
6
  def queues(name)
8
7
  @@queues[name.to_s] ||= Shoryuken::Queue.new(sqs, name)
9
8
  end
10
9
 
11
- def sns
12
- @sns ||= Shoryuken::AwsConfig.sns
13
- end
14
-
15
- def sns_arn
16
- @sns_arn ||= SnsArn
17
- end
18
-
19
10
  def sqs
20
- @sqs ||= Shoryuken::AwsConfig.sqs
11
+ @@sqs ||= Shoryuken.sqs_client
21
12
  end
22
13
 
23
- def topics(name)
24
- @@topics[name.to_s] ||= Topic.new(name, sns)
14
+ def sqs=(sqs)
15
+ @@sqs = sqs
25
16
  end
26
-
27
- attr_accessor :account_id
28
- attr_writer :sns, :sqs, :sqs_resource, :sns_arn
29
17
  end
30
18
  end
31
19
  end
@@ -14,11 +14,15 @@ module Shoryuken
14
14
 
15
15
  def fetch_worker(queue, message)
16
16
  worker_class = !message.is_a?(Array) &&
17
- message.message_attributes &&
18
- message.message_attributes['shoryuken_class'] &&
19
- message.message_attributes['shoryuken_class'][:string_value]
20
-
21
- worker_class = (worker_class.constantize rescue nil) || @workers[queue]
17
+ message.message_attributes &&
18
+ message.message_attributes['shoryuken_class'] &&
19
+ message.message_attributes['shoryuken_class'][:string_value]
20
+
21
+ worker_class = begin
22
+ worker_class.constantize
23
+ rescue
24
+ @workers[queue]
25
+ end
22
26
 
23
27
  worker_class.new
24
28
  end
@@ -28,10 +28,8 @@ module Shoryuken
28
28
  prefix_active_job_queue_names
29
29
  parse_queues
30
30
  require_workers
31
- initialize_aws
32
31
  validate_queues
33
32
  validate_workers
34
- patch_deprecated_workers
35
33
  end
36
34
 
37
35
  private
@@ -49,16 +47,6 @@ module Shoryuken
49
47
  YAML.load(ERB.new(IO.read(path)).result).deep_symbolize_keys
50
48
  end
51
49
 
52
- # DEPRECATED: Please use configure_server and configure_client in
53
- # https://github.com/phstc/shoryuken/blob/a81637d577b36c5cf245882733ea91a335b6602f/lib/shoryuken.rb#L82
54
- # Please delete this method afert next release (v2.0.12 or later)
55
- def initialize_aws
56
- unless Shoryuken.options[:aws].to_h.empty?
57
- Shoryuken.logger.warn { '[DEPRECATION] aws in shoryuken.yml is deprecated. Please use configure_server and configure_client in your initializer' }
58
- end
59
- Shoryuken::AwsConfig.setup(Shoryuken.options[:aws])
60
- end
61
-
62
50
  def initialize_logger
63
51
  Shoryuken::Logging.initialize_logger(Shoryuken.options[:logfile]) if Shoryuken.options[:logfile]
64
52
  Shoryuken.logger.level = Logger::DEBUG if Shoryuken.options[:verbose]
@@ -83,12 +71,11 @@ module Shoryuken
83
71
  end
84
72
 
85
73
  def merge_cli_defined_queues
86
- cli_defined_queues = options.delete(:queues) || []
74
+ cli_defined_queues = options[:queues].to_a
87
75
 
88
76
  cli_defined_queues.each do |cli_defined_queue|
89
- Shoryuken.options[:queues].delete_if do |config_file_queue|
90
- config_file_queue[0] == cli_defined_queue[0]
91
- end
77
+ # CLI defined queues override config_file defined queues
78
+ Shoryuken.options[:queues].delete_if { |config_file_queue| config_file_queue[0] == cli_defined_queue[0] }
92
79
 
93
80
  Shoryuken.options[:queues] << cli_defined_queue
94
81
  end
@@ -110,7 +97,7 @@ module Shoryuken
110
97
  end
111
98
 
112
99
  def parse_queue(queue, weight = nil)
113
- [weight.to_i, 1].max.times { Shoryuken.queues << queue }
100
+ Shoryuken.add_queue(queue, [weight.to_i, 1].max)
114
101
  end
115
102
 
116
103
  def parse_queues
@@ -119,24 +106,6 @@ module Shoryuken
119
106
  end
120
107
  end
121
108
 
122
- def patch_deprecated_workers
123
- Shoryuken.worker_registry.queues.each do |queue|
124
- Shoryuken.worker_registry.workers(queue).each do |worker_class|
125
- if worker_class.instance_method(:perform).arity == 1
126
- Shoryuken.logger.warn { "[DEPRECATION] #{worker_class.name}#perform(sqs_msg) is deprecated. Please use #{worker_class.name}#perform(sqs_msg, body)" }
127
-
128
- worker_class.class_eval do
129
- alias_method :deprecated_perform, :perform
130
-
131
- def perform(sqs_msg, body = nil)
132
- deprecated_perform(sqs_msg)
133
- end
134
- end
135
- end
136
- end
137
- end
138
- end
139
-
140
109
  def require_workers
141
110
  required = Shoryuken.options[:require]
142
111
 
@@ -156,7 +125,7 @@ module Shoryuken
156
125
 
157
126
  Shoryuken.queues.uniq.each do |queue|
158
127
  begin
159
- Shoryuken::Client.queues queue
128
+ Shoryuken::Client.queues(queue)
160
129
  rescue Aws::SQS::Errors::NonExistentQueue
161
130
  non_existent_queues << queue
162
131
  end
@@ -166,13 +135,13 @@ module Shoryuken
166
135
  end
167
136
 
168
137
  def validate_workers
138
+ return if defined?(::ActiveJob)
139
+
169
140
  all_queues = Shoryuken.queues
170
141
  queues_with_workers = Shoryuken.worker_registry.queues
171
142
 
172
- unless defined?(::ActiveJob)
173
- (all_queues - queues_with_workers).each do |queue|
174
- Shoryuken.logger.warn { "No worker supplied for '#{queue}'" }
175
- end
143
+ (all_queues - queues_with_workers).each do |queue|
144
+ Shoryuken.logger.warn { "No worker supplied for '#{queue}'" }
176
145
  end
177
146
  end
178
147
  end
@@ -5,23 +5,21 @@ module Shoryuken
5
5
  FETCH_LIMIT = 10
6
6
 
7
7
  def fetch(queue, available_processors)
8
- watchdog('Fetcher#fetch died') do
9
- started_at = Time.now
10
-
11
- logger.debug { "Looking for new messages in '#{queue}'" }
12
-
13
- begin
14
- limit = available_processors > FETCH_LIMIT ? FETCH_LIMIT : available_processors
15
-
16
- sqs_msgs = Array(receive_messages(queue, limit))
17
- logger.info { "Found #{sqs_msgs.size} messages for '#{queue.name}'" } if !sqs_msgs.empty?
18
- logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
19
- sqs_msgs
20
- rescue => ex
21
- logger.error { "Error fetching message: #{ex}" }
22
- logger.error { ex.backtrace.first }
23
- []
24
- end
8
+ started_at = Time.now
9
+
10
+ logger.debug { "Looking for new messages in '#{queue}'" }
11
+
12
+ begin
13
+ limit = available_processors > FETCH_LIMIT ? FETCH_LIMIT : available_processors
14
+
15
+ sqs_msgs = Array(receive_messages(queue, limit))
16
+ logger.info { "Found #{sqs_msgs.size} messages for '#{queue.name}'" } unless sqs_msgs.empty?
17
+ logger.debug { "Fetcher for '#{queue}' completed in #{elapsed(started_at)} ms" }
18
+ sqs_msgs
19
+ rescue => ex
20
+ logger.error { "Error fetching message: #{ex}" }
21
+ logger.error { ex.backtrace.first }
22
+ []
25
23
  end
26
24
  end
27
25
 
@@ -31,7 +29,7 @@ module Shoryuken
31
29
  # AWS limits the batch size by 10
32
30
  limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
33
31
 
34
- options = (Shoryuken.options[:aws][:receive_message] || {}).dup
32
+ options = Shoryuken.sqs_client_receive_message_opts.to_h.dup
35
33
  options[:max_number_of_messages] = limit
36
34
  options[:message_attribute_names] = %w(All)
37
35
  options[:attribute_names] = %w(All)