shoryuken 3.0.11 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -24,7 +24,7 @@ module Shoryuken
24
24
  @workers[queue]
25
25
  end
26
26
 
27
- worker_class.new
27
+ worker_class.new if worker_class
28
28
  end
29
29
 
30
30
  def queues
@@ -34,7 +34,7 @@ module Shoryuken
34
34
  def register_worker(queue, clazz)
35
35
  if (worker_class = @workers[queue])
36
36
  if worker_class.get_shoryuken_options['batch'] == true || clazz.get_shoryuken_options['batch'] == true
37
- fail ArgumentError, "Could not register #{clazz} for '#{queue}', "\
37
+ fail ArgumentError, "Could not register #{clazz} for #{queue}, "\
38
38
  "because #{worker_class} is already registered for this queue, "\
39
39
  "and Shoryuken doesn't support a batchable worker for a queue with multiple workers"
40
40
  end
@@ -44,7 +44,11 @@ module Shoryuken
44
44
 
45
45
  fail ArgumentError, "The supplied config file #{path} does not exist" unless File.exist?(path)
46
46
 
47
- YAML.load(ERB.new(IO.read(path)).result).deep_symbolize_keys
47
+ if result = YAML.load(ERB.new(IO.read(path)).result)
48
+ result.deep_symbolize_keys
49
+ else
50
+ {}
51
+ end
48
52
  end
49
53
 
50
54
  def initialize_logger
@@ -96,13 +100,24 @@ module Shoryuken
96
100
  end
97
101
  end
98
102
 
99
- def parse_queue(queue, weight = nil)
100
- Shoryuken.add_queue(queue, [weight.to_i, 1].max)
103
+ def parse_queue(queue, weight = nil, group)
104
+ Shoryuken.add_queue(queue, [weight.to_i, 1].max, group)
101
105
  end
102
106
 
103
107
  def parse_queues
104
- Shoryuken.options[:queues].to_a.each do |queue_and_weight|
105
- parse_queue(*queue_and_weight)
108
+ if Shoryuken.options[:queues].to_a.any?
109
+ Shoryuken.add_group('default', Shoryuken.options.fetch(:concurrency, 25))
110
+
111
+ Shoryuken.options[:queues].to_a.each do |queue, weight|
112
+ parse_queue(queue, weight, 'default')
113
+ end
114
+ end
115
+
116
+ Shoryuken.options[:groups].to_a.each do |group, options|
117
+ Shoryuken.add_group(group, options.fetch(:concurrency, 25))
118
+ options[:queues].to_a.each do |queue, weight|
119
+ parse_queue(queue, weight, group)
120
+ end
106
121
  end
107
122
  end
108
123
 
@@ -119,10 +134,11 @@ module Shoryuken
119
134
  end
120
135
 
121
136
  def validate_queues
122
- return Shoryuken.logger.warn { 'No queues supplied' } if Shoryuken.queues.empty?
137
+ return Shoryuken.logger.warn { 'No queues supplied' } if Shoryuken.ungrouped_queues.empty?
138
+
123
139
  non_existent_queues = []
124
140
 
125
- Shoryuken.queues.uniq.each do |queue|
141
+ Shoryuken.ungrouped_queues.uniq.each do |queue|
126
142
  begin
127
143
  Shoryuken::Client.queues(queue)
128
144
  rescue Aws::Errors::NoSuchEndpointError, Aws::SQS::Errors::NonExistentQueue
@@ -134,14 +150,14 @@ module Shoryuken
134
150
 
135
151
  fail(
136
152
  ArgumentError,
137
- "The specified queue(s) #{non_existent_queues.join(', ')} do not exist.\nCheck 'shoryuken sqs create QUEUE-NAME' for creating a queue with default settings"
153
+ "The specified queue(s) #{non_existent_queues.join(', ')} do not exist.\nTry 'shoryuken sqs create QUEUE-NAME' for creating a queue with default settings"
138
154
  )
139
155
  end
140
156
 
141
157
  def validate_workers
142
158
  return if defined?(::ActiveJob)
143
159
 
144
- all_queues = Shoryuken.queues
160
+ all_queues = Shoryuken.ungrouped_queues
145
161
  queues_with_workers = Shoryuken.worker_registry.queues
146
162
 
147
163
  (all_queues - queues_with_workers).each do |queue|
@@ -4,23 +4,23 @@ module Shoryuken
4
4
 
5
5
  FETCH_LIMIT = 10
6
6
 
7
- def fetch(queue, available_processors)
7
+ attr_reader :group
8
+
9
+ def initialize(group)
10
+ @group = group
11
+ end
12
+
13
+ def fetch(queue, limit)
8
14
  started_at = Time.now
9
15
 
10
16
  logger.debug { "Looking for new messages in #{queue}" }
11
17
 
12
- begin
13
- limit = available_processors > FETCH_LIMIT ? FETCH_LIMIT : available_processors
14
-
15
- sqs_msgs = Array(receive_messages(queue, limit))
16
- logger.info { "Found #{sqs_msgs.size} messages for #{queue.name}" } unless sqs_msgs.empty?
17
- logger.debug { "Fetcher for #{queue} completed in #{elapsed(started_at)} ms" }
18
- sqs_msgs
19
- rescue => ex
20
- logger.error { "Error fetching message: #{ex.message}" }
21
- logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
22
- []
23
- end
18
+ sqs_msgs = Array(receive_messages(queue, [FETCH_LIMIT, limit].min))
19
+
20
+ logger.info { "Found #{sqs_msgs.size} messages for #{queue.name}" } unless sqs_msgs.empty?
21
+ logger.debug { "Fetcher for #{queue} completed in #{elapsed(started_at)} ms" }
22
+
23
+ sqs_msgs
24
24
  end
25
25
 
26
26
  private
@@ -29,10 +29,11 @@ module Shoryuken
29
29
  # AWS limits the batch size by 10
30
30
  limit = limit > FETCH_LIMIT ? FETCH_LIMIT : limit
31
31
 
32
- options = Shoryuken.sqs_client_receive_message_opts.to_h.dup
33
- options[:max_number_of_messages] = limit
32
+ options = Shoryuken.sqs_client_receive_message_opts[group].to_h.dup
33
+
34
+ options[:max_number_of_messages] = limit
34
35
  options[:message_attribute_names] = %w(All)
35
- options[:attribute_names] = %w(All)
36
+ options[:attribute_names] = %w(All)
36
37
 
37
38
  options.merge!(queue.options)
38
39
 
@@ -3,17 +3,96 @@ module Shoryuken
3
3
  include Util
4
4
 
5
5
  def initialize
6
- @manager = Shoryuken::Manager.new(Shoryuken::Fetcher.new,
7
- Shoryuken.options[:polling_strategy].new(Shoryuken.queues))
6
+ @managers = create_managers
7
+ @shutdowning = Concurrent::AtomicBoolean.new(false)
8
8
  end
9
9
 
10
- def stop(options = {})
11
- @manager.stop(shutdown: !options[:shutdown].nil?,
12
- timeout: Shoryuken.options[:timeout])
10
+ def start
11
+ logger.info { 'Starting' }
12
+
13
+ start_callback
14
+ start_managers
15
+ end
16
+
17
+ def stop!
18
+ initiate_stop
19
+
20
+ executor.shutdown
21
+
22
+ return if executor.wait_for_termination(Shoryuken.options[:timeout])
23
+
24
+ executor.kill
25
+ end
26
+
27
+ def stop
28
+ fire_event(:quiet, true)
29
+
30
+ initiate_stop
31
+
32
+ executor.shutdown
33
+ executor.wait_for_termination
34
+ end
35
+
36
+ private
37
+
38
+ def executor
39
+ Concurrent.global_io_executor
40
+ end
41
+
42
+ def start_managers
43
+ @managers.each do |manager|
44
+ Concurrent::Promise.execute { manager.start }.rescue do |ex|
45
+ log_manager_failure(ex)
46
+ start_soft_shutdown
47
+ end
48
+ end
49
+ end
50
+
51
+ def start_soft_shutdown
52
+ Process.kill('USR1', Process.pid) if @shutdowning.make_true
53
+ end
54
+
55
+ def log_manager_failure(ex)
56
+ return unless ex
57
+
58
+ logger.error { "Manager failed: #{ex.message}" }
59
+ logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
60
+ end
61
+
62
+ def initiate_stop
63
+ logger.info { 'Shutting down' }
64
+
65
+ @managers.each(&:stop)
66
+
67
+ stop_callback
68
+ end
69
+
70
+ def start_callback
71
+ if (callback = Shoryuken.start_callback)
72
+ logger.debug { 'Calling start_callback' }
73
+ callback.call
74
+ end
75
+
76
+ fire_event(:startup)
77
+ end
78
+
79
+ def stop_callback
80
+ if (callback = Shoryuken.stop_callback)
81
+ logger.debug { 'Calling stop_callback' }
82
+ callback.call
83
+ end
84
+
85
+ fire_event(:shutdown, true)
13
86
  end
14
87
 
15
- def run
16
- @manager.start
88
+ def create_managers
89
+ Shoryuken.groups.map do |group, options|
90
+ Shoryuken::Manager.new(
91
+ Shoryuken::Fetcher.new(group),
92
+ Shoryuken.polling_strategy(group).new(options[:queues]),
93
+ options[:concurrency]
94
+ )
95
+ end
17
96
  end
18
97
  end
19
98
  end
@@ -6,95 +6,71 @@ module Shoryuken
6
6
  # See https://github.com/phstc/shoryuken/issues/348#issuecomment-292847028
7
7
  MIN_DISPATCH_INTERVAL = 0.1
8
8
 
9
- def initialize(fetcher, polling_strategy)
10
- @count = Shoryuken.options.fetch(:concurrency, 25)
11
-
12
- raise(ArgumentError, "Concurrency value #{@count} is invalid, it needs to be a positive number") unless @count > 0
13
-
14
- @queues = Shoryuken.queues.dup.uniq
15
-
16
- @done = Concurrent::AtomicBoolean.new(false)
17
-
18
- @fetcher = fetcher
9
+ def initialize(fetcher, polling_strategy, concurrency)
10
+ @fetcher = fetcher
19
11
  @polling_strategy = polling_strategy
20
-
21
- @pool = Concurrent::FixedThreadPool.new(@count, max_queue: @count)
22
- @dispatcher_executor = Concurrent::SingleThreadExecutor.new
12
+ @max_processors = concurrency
13
+ @busy_processors = Concurrent::AtomicFixnum.new(0)
14
+ @done = Concurrent::AtomicBoolean.new(false)
23
15
  end
24
16
 
25
17
  def start
26
- logger.info { 'Starting' }
27
-
28
- dispatch_async
18
+ dispatch
29
19
  end
30
20
 
31
- def stop(options = {})
21
+ def stop
32
22
  @done.make_true
33
-
34
- if (callback = Shoryuken.stop_callback)
35
- logger.info { 'Calling on_stop callback' }
36
- callback.call
37
- end
38
-
39
- fire_event(:shutdown, true)
40
-
41
- logger.info { 'Shutting down workers' }
42
-
43
- @dispatcher_executor.kill
44
-
45
- if options[:shutdown]
46
- hard_shutdown_in(options[:timeout])
47
- else
48
- soft_shutdown
49
- end
50
23
  end
51
24
 
52
- def processor_failed(ex)
53
- logger.error { "Processor failed: #{ex.message}" }
54
- logger.error { ex.backtrace.join("\n") } unless ex.backtrace.nil?
55
- end
25
+ private
56
26
 
57
- def processor_done(queue)
58
- logger.debug { "Process done for #{queue}" }
27
+ def stopped?
28
+ @done.true? || !Concurrent.global_io_executor.running?
59
29
  end
60
30
 
61
- private
31
+ def dispatch
32
+ return if stopped?
62
33
 
63
- def dispatch_async
64
- @dispatcher_executor.post(&method(:dispatch_now))
65
- end
34
+ if !ready.positive? || (queue = @polling_strategy.next_queue).nil?
35
+ return dispatch_later
36
+ end
66
37
 
67
- def dispatch_now
68
- return if @done.true?
38
+ fire_event(:dispatch)
69
39
 
70
- begin
71
- if ready.zero? || (queue = @polling_strategy.next_queue).nil?
72
- sleep MIN_DISPATCH_INTERVAL
73
- return
74
- end
40
+ logger.info { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{@polling_strategy.active_queues}" }
75
41
 
76
- fire_event(:dispatch)
42
+ batched_queue?(queue) ? dispatch_batch(queue) : dispatch_single_messages(queue)
77
43
 
78
- logger.debug { "Ready: #{ready}, Busy: #{busy}, Active Queues: #{@polling_strategy.active_queues}" }
44
+ dispatch
45
+ end
79
46
 
80
- batched_queue?(queue) ? dispatch_batch(queue) : dispatch_single_messages(queue)
81
- ensure
82
- dispatch_async
83
- end
47
+ def dispatch_later
48
+ sleep(MIN_DISPATCH_INTERVAL)
49
+ dispatch
84
50
  end
85
51
 
86
52
  def busy
87
- @count - ready
53
+ @busy_processors.value
88
54
  end
89
55
 
90
56
  def ready
91
- @pool.remaining_capacity
57
+ @max_processors - busy
92
58
  end
93
59
 
94
- def assign(queue, sqs_msg)
60
+ def processor_done
61
+ @busy_processors.decrement
62
+ end
63
+
64
+ def assign(queue_name, sqs_msg)
65
+ return if stopped?
66
+
95
67
  logger.debug { "Assigning #{sqs_msg.message_id}" }
96
68
 
97
- @pool.post { Processor.new(self).process(queue, sqs_msg) }
69
+ @busy_processors.increment
70
+
71
+ Concurrent::Promise.execute {
72
+ Processor.new(queue_name, sqs_msg).process
73
+ }.then { processor_done }.rescue { processor_done }
98
74
  end
99
75
 
100
76
  def dispatch_batch(queue)
@@ -105,6 +81,7 @@ module Shoryuken
105
81
 
106
82
  def dispatch_single_messages(queue)
107
83
  messages = @fetcher.fetch(queue, ready)
84
+
108
85
  @polling_strategy.messages_found(queue.name, messages.size)
109
86
  messages.each { |message| assign(queue.name, message) }
110
87
  end
@@ -113,24 +90,6 @@ module Shoryuken
113
90
  Shoryuken.worker_registry.batch_receive_messages?(queue.name)
114
91
  end
115
92
 
116
- def soft_shutdown
117
- @pool.shutdown
118
- @pool.wait_for_termination
119
- end
120
-
121
- def hard_shutdown_in(delay)
122
- if busy > 0
123
- logger.info { "Pausing up to #{delay} seconds to allow workers to finish..." }
124
- end
125
-
126
- @pool.shutdown
127
-
128
- return if @pool.wait_for_termination(delay)
129
-
130
- logger.info { "Hard shutting down #{busy} busy workers" }
131
- @pool.kill
132
- end
133
-
134
93
  def patch_batch!(sqs_msgs)
135
94
  sqs_msgs.instance_eval do
136
95
  def message_id
@@ -0,0 +1,192 @@
1
+ module Shoryuken
2
+ class Options
3
+ DEFAULTS = {
4
+ concurrency: 25,
5
+ queues: [],
6
+ aws: {},
7
+ delay: 0,
8
+ timeout: 8,
9
+ lifecycle_events: {
10
+ startup: [],
11
+ dispatch: [],
12
+ quiet: [],
13
+ shutdown: []
14
+ }
15
+ }.freeze
16
+
17
+ @@groups = {}
18
+ @@worker_registry = DefaultWorkerRegistry.new
19
+ @@active_job_queue_name_prefixing = false
20
+ @@sqs_client = nil
21
+ @@sqs_client_receive_message_opts = {}
22
+ @@start_callback = nil
23
+ @@stop_callback = nil
24
+
25
+ class << self
26
+ def add_group(group, concurrency)
27
+ groups[group] ||= {
28
+ concurrency: concurrency,
29
+ queues: []
30
+ }
31
+ end
32
+
33
+ def groups
34
+ @@groups
35
+ end
36
+
37
+ def add_queue(queue, weight, group)
38
+ weight.times do
39
+ groups[group][:queues] << queue
40
+ end
41
+ end
42
+
43
+ def ungrouped_queues
44
+ groups.values.flat_map { |options| options[:queues] }
45
+ end
46
+
47
+ def worker_registry
48
+ @@worker_registry
49
+ end
50
+
51
+ def worker_registry=(worker_registry)
52
+ @@worker_registry = worker_registry
53
+ end
54
+
55
+ def polling_strategy(group)
56
+ options[group].to_h.fetch(:polling_strategy, Polling::WeightedRoundRobin)
57
+ end
58
+
59
+ def start_callback
60
+ @@start_callback
61
+ end
62
+
63
+ def start_callback=(start_callback)
64
+ @@start_callback = start_callback
65
+ end
66
+
67
+ def stop_callback
68
+ @@stop_callback
69
+ end
70
+
71
+ def stop_callback=(stop_callback)
72
+ @@stop_callback = stop_callback
73
+ end
74
+
75
+ def active_job_queue_name_prefixing
76
+ @@active_job_queue_name_prefixing
77
+ end
78
+
79
+ def active_job_queue_name_prefixing=(active_job_queue_name_prefixing)
80
+ @@active_job_queue_name_prefixing = active_job_queue_name_prefixing
81
+ end
82
+
83
+ def sqs_client
84
+ @@sqs_client ||= Aws::SQS::Client.new
85
+ end
86
+
87
+ def sqs_client=(sqs_client)
88
+ @@sqs_client = sqs_client
89
+ end
90
+
91
+ def sqs_client_receive_message_opts
92
+ @@sqs_client_receive_message_opts
93
+ end
94
+
95
+ def sqs_client_receive_message_opts=(sqs_client_receive_message_opts)
96
+ @@sqs_client_receive_message_opts['default'] = sqs_client_receive_message_opts
97
+ end
98
+
99
+ def options
100
+ @@options ||= DEFAULTS.dup
101
+ end
102
+
103
+ def logger
104
+ Shoryuken::Logging.logger
105
+ end
106
+
107
+ def register_worker(*args)
108
+ @@worker_registry.register_worker(*args)
109
+ end
110
+
111
+ def configure_server
112
+ yield self if server?
113
+ end
114
+
115
+ def server_middleware
116
+ @@server_chain ||= default_server_middleware
117
+ yield @@server_chain if block_given?
118
+ @@server_chain
119
+ end
120
+
121
+ def configure_client
122
+ yield self unless server?
123
+ end
124
+
125
+ def client_middleware
126
+ @@client_chain ||= default_client_middleware
127
+ yield @@client_chain if block_given?
128
+ @@client_chain
129
+ end
130
+
131
+ def default_worker_options
132
+ @@default_worker_options ||= {
133
+ 'queue' => 'default',
134
+ 'delete' => false,
135
+ 'auto_delete' => false,
136
+ 'auto_visibility_timeout' => false,
137
+ 'retry_intervals' => nil,
138
+ 'batch' => false
139
+ }
140
+ end
141
+
142
+ def default_worker_options=(default_worker_options)
143
+ @@default_worker_options = default_worker_options
144
+ end
145
+
146
+ def on_start(&block)
147
+ @@start_callback = block
148
+ end
149
+
150
+ def on_stop(&block)
151
+ @@stop_callback = block
152
+ end
153
+
154
+ # Register a block to run at a point in the Shoryuken lifecycle.
155
+ # :startup, :quiet or :shutdown are valid events.
156
+ #
157
+ # Shoryuken.configure_server do |config|
158
+ # config.on(:shutdown) do
159
+ # puts "Goodbye cruel world!"
160
+ # end
161
+ # end
162
+ def on(event, &block)
163
+ fail ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
164
+ fail ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
165
+ options[:lifecycle_events][event] << block
166
+ end
167
+
168
+ private
169
+
170
+ def default_server_middleware
171
+ Middleware::Chain.new do |m|
172
+ m.add Middleware::Server::Timing
173
+ m.add Middleware::Server::ExponentialBackoffRetry
174
+ m.add Middleware::Server::AutoDelete
175
+ m.add Middleware::Server::AutoExtendVisibility
176
+ if defined?(::ActiveRecord::Base)
177
+ require 'shoryuken/middleware/server/active_record'
178
+ m.add Middleware::Server::ActiveRecord
179
+ end
180
+ end
181
+ end
182
+
183
+ def default_client_middleware
184
+ Middleware::Chain.new
185
+ end
186
+
187
+ def server?
188
+ defined?(Shoryuken::CLI)
189
+ end
190
+ end
191
+ end
192
+ end