shoryuken 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,23 @@
1
+
2
+ module Shoryuken
3
+ class Client
4
+ @@queues = {}
5
+
6
+ def self.queues(name)
7
+ @@queues[name.to_s] ||= sqs.queues.named(name)
8
+ end
9
+
10
+ def self.receive_message(queue, options = {})
11
+ queues(queue).receive_message(Hash(options))
12
+ end
13
+
14
+ def self.reset!
15
+ # for test purposes
16
+ @@queues = {}
17
+ end
18
+
19
+ def self.sqs
20
+ @sqs ||= AWS::SQS.new
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,47 @@
1
+ begin
2
+ require 'active_support/core_ext/hash/keys'
3
+ require 'active_support/core_ext/hash/deep_merge'
4
+ rescue LoadError
5
+ class Hash
6
+ def stringify_keys
7
+ keys.each do |key|
8
+ self[key.to_s] = delete(key)
9
+ end
10
+ self
11
+ end if !{}.respond_to?(:stringify_keys)
12
+
13
+ def symbolize_keys
14
+ keys.each do |key|
15
+ self[(key.to_sym rescue key) || key] = delete(key)
16
+ end
17
+ self
18
+ end if !{}.respond_to?(:symbolize_keys)
19
+
20
+ def deep_symbolize_keys
21
+ keys.each do |key|
22
+ value = delete(key)
23
+ self[(key.to_sym rescue key) || key] = value
24
+
25
+ value.deep_symbolize_keys if value.is_a? Hash
26
+ end
27
+ self
28
+ end if !{}.respond_to?(:deep_symbolize_keys)
29
+ end
30
+ end
31
+
32
+ begin
33
+ require 'active_support/core_ext/string/inflections'
34
+ rescue LoadError
35
+ class String
36
+ def constantize
37
+ names = self.split('::')
38
+ names.shift if names.empty? || names.first.empty?
39
+
40
+ constant = Object
41
+ names.each do |name|
42
+ constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
43
+ end
44
+ constant
45
+ end
46
+ end if !"".respond_to?(:constantize)
47
+ end
@@ -0,0 +1,49 @@
1
+ module Shoryuken
2
+ class Fetcher
3
+ include Celluloid
4
+ include Util
5
+
6
+ def initialize(manager)
7
+ @manager = manager
8
+ end
9
+
10
+ def receive_message(queue, limit)
11
+ # AWS limits the batch size by 10
12
+ limit = limit > 10 ? 10 : limit
13
+
14
+ Shoryuken::Client.receive_message queue, Shoryuken.options[:aws][:receive_message].to_h.merge(limit: limit)
15
+ end
16
+
17
+ def fetch(queue, available_processors)
18
+ watchdog('Fetcher#fetch died') do
19
+ started_at = Time.now
20
+
21
+ logger.info "Looking for new messages queue '#{queue}'"
22
+
23
+ begin
24
+ if (sqs_msgs = Array(receive_message(queue, available_processors))).any?
25
+ logger.info "Message found for queue '#{queue}'"
26
+
27
+ sqs_msgs.each do |sqs_msg|
28
+ @manager.async.rebalance_queue_weight!(queue)
29
+ @manager.async.assign(queue, sqs_msg)
30
+ end
31
+ else
32
+ logger.info "No message found for queue '#{queue}'"
33
+
34
+ @manager.async.pause_queue!(queue)
35
+ end
36
+
37
+ @manager.async.dispatch
38
+
39
+ logger.debug "Fetcher#fetch('#{queue}') completed in #{elapsed(started_at)} ms"
40
+ rescue => ex
41
+ logger.error("Error fetching message: #{ex}")
42
+ logger.error(ex.backtrace.first)
43
+
44
+ @manager.async.dispatch
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,41 @@
1
+ module Shoryuken
2
+ class Launcher
3
+ include Celluloid
4
+ include Util
5
+
6
+ trap_exit :actor_died
7
+
8
+ attr_accessor :manager
9
+
10
+ def initialize
11
+ @manager = Shoryuken::Manager.new_link
12
+ @fetcher = Shoryuken::Fetcher.new_link(manager)
13
+
14
+ @done = false
15
+
16
+ manager.fetcher = @fetcher
17
+ end
18
+
19
+ def stop(options = {})
20
+ watchdog('Launcher#stop') do
21
+ @done = true
22
+ @fetcher.terminate if @fetcher.alive?
23
+
24
+ manager.async.stop(shutdown: !!options[:shutdown], timeout: Shoryuken.options[:timeout])
25
+ manager.wait(:shutdown)
26
+ end
27
+ end
28
+
29
+ def run
30
+ watchdog('Launcher#run') do
31
+ manager.async.start
32
+ end
33
+ end
34
+
35
+ def actor_died(actor, reason)
36
+ return if @done
37
+ Shoryuken.logger.warn 'Shoryuken died due to the following error, cannot recover, process exiting'
38
+ exit 1
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,47 @@
1
+ require 'time'
2
+ require 'logger'
3
+
4
+ module Shoryuken
5
+ module Logging
6
+
7
+ class Pretty < Logger::Formatter
8
+ # Provide a call() method that returns the formatted message.
9
+ def call(severity, time, program_name, message)
10
+ "#{time.utc.iso8601} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
11
+ end
12
+
13
+ def context
14
+ c = Thread.current[:shoryuken_context]
15
+ c ? " #{c}" : ''
16
+ end
17
+ end
18
+
19
+ def self.with_context(msg)
20
+ begin
21
+ Thread.current[:shoryuken_context] = msg
22
+ yield
23
+ ensure
24
+ Thread.current[:shoryuken_context] = nil
25
+ end
26
+ end
27
+
28
+ def self.initialize_logger(log_target = STDOUT)
29
+ @logger = Logger.new(log_target)
30
+ @logger.level = Logger::INFO
31
+ @logger.formatter = Pretty.new
32
+ @logger
33
+ end
34
+
35
+ def self.logger
36
+ @logger || initialize_logger
37
+ end
38
+
39
+ def self.logger=(log)
40
+ @logger = (log ? log : Logger.new('/dev/null'))
41
+ end
42
+
43
+ def logger
44
+ shoryuken::Logging.logger
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,220 @@
1
+ require 'yaml'
2
+ require 'aws-sdk'
3
+ require 'celluloid'
4
+
5
+ require 'shoryuken/version'
6
+ require 'shoryuken/manager'
7
+ require 'shoryuken/processor'
8
+ require 'shoryuken/fetcher'
9
+
10
+ module Shoryuken
11
+ class Manager
12
+ include Celluloid
13
+ include Util
14
+
15
+ attr_accessor :fetcher
16
+
17
+ trap_exit :processor_died
18
+
19
+ def initialize
20
+ @count = Shoryuken.options[:concurrency] || 25
21
+ @queues = Shoryuken.queues.dup.uniq
22
+
23
+ @done = false
24
+
25
+ @busy = []
26
+ @ready = @count.times.map { Processor.new_link(current_actor) }
27
+ end
28
+
29
+ def start
30
+ logger.info 'Starting'
31
+
32
+ dispatch
33
+ end
34
+
35
+ def stop(options = {})
36
+ watchdog('Manager#stop died') do
37
+ @done = true
38
+
39
+ @fetcher.terminate if @fetcher.alive?
40
+
41
+ logger.info { "Shutting down #{@ready.size} quiet workers" }
42
+
43
+ @ready.each do |processor|
44
+ processor.terminate if processor.alive?
45
+ end
46
+ @ready.clear
47
+
48
+ return after(0) { signal(:shutdown) } if @busy.empty?
49
+
50
+ if options[:shutdown]
51
+ hard_shutdown_in(options[:timeout])
52
+ else
53
+ soft_shutdown(options[:timeout])
54
+ end
55
+ end
56
+ end
57
+
58
+ def processor_done(queue, processor)
59
+ watchdog('Manager#processor_done died') do
60
+ logger.info "Process done for queue '#{queue}'"
61
+
62
+ @busy.delete processor
63
+
64
+ if stopped?
65
+ processor.terminate if processor.alive?
66
+ else
67
+ @ready << processor
68
+ end
69
+ end
70
+ end
71
+
72
+ def processor_died(processor, reason)
73
+ watchdog("Manager#processor_died died") do
74
+ logger.info "Process died, reason: #{reason}"
75
+
76
+ @busy.delete processor
77
+
78
+ unless stopped?
79
+ @ready << Processor.new_link(current_actor)
80
+ end
81
+ end
82
+ end
83
+
84
+ def stopped?
85
+ @done
86
+ end
87
+
88
+ def assign(queue, sqs_msg)
89
+ watchdog("Manager#assign died") do
90
+ logger.info "Assigning #{sqs_msg.id}"
91
+
92
+ processor = @ready.pop
93
+ @busy << processor
94
+
95
+ processor.async.process(queue, sqs_msg)
96
+ end
97
+ end
98
+
99
+ def rebalance_queue_weight!(queue)
100
+ watchdog('Manager#rebalance_queue_weight! died') do
101
+ if (original = original_queue_weight(queue)) > (current = current_queue_weight(queue))
102
+ if current + 1 > original
103
+ logger.info "Increasing queue '#{queue}' weight to #{current + 1}, original: #{original}"
104
+ else
105
+ logger.info "Queue '#{queue}' is back to its normal weight #{original}"
106
+ end
107
+
108
+ @queues << queue
109
+ end
110
+ end
111
+ end
112
+
113
+ def pause_queue!(queue)
114
+ return unless @queues.include? queue
115
+
116
+ logger.info "Pausing queue '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because the queue is empty"
117
+
118
+ @queues.delete(queue)
119
+
120
+ after(Shoryuken.options[:delay].to_f) { async.restart_queue!(queue) }
121
+ end
122
+
123
+
124
+ def dispatch
125
+ return if stopped?
126
+
127
+ logger.debug { "Ready size: #{@ready.size}" }
128
+ logger.debug { "Busy size: #{@busy.size}" }
129
+ logger.debug { "Queues: #{@queues.inspect}" }
130
+
131
+ if @ready.empty?
132
+ logger.debug { 'Pausing fetcher, because all processors are busy' }
133
+
134
+ after(1) { dispatch }
135
+
136
+ return
137
+ end
138
+
139
+ if queue = next_queue
140
+ @fetcher.async.fetch(queue, @ready.size)
141
+ else
142
+ logger.debug { 'Pausing fetcher, because all queues are paused' }
143
+
144
+ @fetcher_paused = true
145
+ end
146
+ end
147
+
148
+ private
149
+
150
+ def restart_queue!(queue)
151
+ return if stopped?
152
+
153
+ unless @queues.include? queue
154
+ logger.info "Restarting queue '#{queue}'"
155
+
156
+ @queues << queue
157
+
158
+ if @fetcher_paused
159
+ logger.debug { 'Restarting fetcher' }
160
+
161
+ @fetcher_paused = false
162
+
163
+
164
+ dispatch
165
+ end
166
+ end
167
+ end
168
+
169
+ def current_queue_weight(queue)
170
+ queue_weight(@queues, queue)
171
+ end
172
+
173
+ def original_queue_weight(queue)
174
+ queue_weight(Shoryuken.queues, queue)
175
+ end
176
+
177
+ def queue_weight(queues, queue)
178
+ queues.count { |q| q == queue }
179
+ end
180
+
181
+ def next_queue
182
+ return nil if @queues.empty?
183
+
184
+ queue = @queues.shift
185
+ @queues << queue
186
+
187
+ queue
188
+ end
189
+
190
+ def soft_shutdown(delay)
191
+ logger.info { "Waiting for #{@busy.size} busy workers" }
192
+
193
+ if @busy.size > 0
194
+ after(delay) { soft_shutdown(delay) }
195
+ else
196
+ after(0) { signal(:shutdown) }
197
+ end
198
+ end
199
+
200
+ def hard_shutdown_in(delay)
201
+ logger.info { "Waiting for #{@busy.size} busy workers" }
202
+ logger.info { "Pausing up to #{delay} seconds to allow workers to finish..." }
203
+
204
+ after(delay) do
205
+ watchdog("Manager#hard_shutdown_in died") do
206
+ if @busy.size > 0
207
+ logger.info { "Hard shutting down #{@busy.size} busy workers" }
208
+
209
+ @busy.each do |processor|
210
+ t = processor.bare_object.actual_work_thread
211
+ t.raise Shutdown if processor.alive?
212
+ end
213
+ end
214
+
215
+ after(0) { signal(:shutdown) }
216
+ end
217
+ end
218
+ end
219
+ end
220
+ end
@@ -0,0 +1,111 @@
1
+ module Shoryuken
2
+ # Middleware is code configured to run before/after
3
+ # a message is processed. It is patterned after Rack
4
+ # middleware. Middleware exists for the server
5
+ # side (when jobs are actually processed).
6
+ #
7
+ # To modify middleware for the server, just call
8
+ # with another block:
9
+ #
10
+ # Shoryuken.configure_server do |config|
11
+ # config.server_middleware do |chain|
12
+ # chain.add MyServerHook
13
+ # chain.remove ActiveRecord
14
+ # end
15
+ # end
16
+ #
17
+ # To insert immediately preceding another entry:
18
+ #
19
+ # Shoryuken.configure_server do |config|
20
+ # config.server_middleware do |chain|
21
+ # chain.insert_before ActiveRecord, MyServerHook
22
+ # end
23
+ # end
24
+ #
25
+ # To insert immediately after another entry:
26
+ #
27
+ # Shoryuken.configure_server do |config|
28
+ # config.server_middleware do |chain|
29
+ # chain.insert_after ActiveRecord, MyServerHook
30
+ # end
31
+ # end
32
+ #
33
+ # This is an example of a minimal server middleware:
34
+ #
35
+ # class MyServerHook
36
+ # def call(worker_instance, queue, sqs_msg)
37
+ # puts 'Before work'
38
+ # yield
39
+ # puts 'After work'
40
+ # end
41
+ # end
42
+ #
43
+ module Middleware
44
+ class Chain
45
+ attr_reader :entries
46
+
47
+ def initialize
48
+ @entries = []
49
+ yield self if block_given?
50
+ end
51
+
52
+ def remove(klass)
53
+ entries.delete_if { |entry| entry.klass == klass }
54
+ end
55
+
56
+ def add(klass, *args)
57
+ entries << Entry.new(klass, *args) unless exists?(klass)
58
+ end
59
+
60
+ def insert_before(oldklass, newklass, *args)
61
+ i = entries.index { |entry| entry.klass == newklass }
62
+ new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
63
+ i = entries.find_index { |entry| entry.klass == oldklass } || 0
64
+ entries.insert(i, new_entry)
65
+ end
66
+
67
+ def insert_after(oldklass, newklass, *args)
68
+ i = entries.index { |entry| entry.klass == newklass }
69
+ new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
70
+ i = entries.find_index { |entry| entry.klass == oldklass } || entries.count - 1
71
+ entries.insert(i+1, new_entry)
72
+ end
73
+
74
+ def exists?(klass)
75
+ entries.any? { |entry| entry.klass == klass }
76
+ end
77
+
78
+ def retrieve
79
+ entries.map(&:make_new)
80
+ end
81
+
82
+ def clear
83
+ entries.clear
84
+ end
85
+
86
+ def invoke(*args, &final_action)
87
+ chain = retrieve.dup
88
+ traverse_chain = lambda do
89
+ if chain.empty?
90
+ final_action.call
91
+ else
92
+ chain.shift.call(*args, &traverse_chain)
93
+ end
94
+ end
95
+ traverse_chain.call
96
+ end
97
+ end
98
+
99
+ class Entry
100
+ attr_reader :klass
101
+ def initialize(klass, *args)
102
+ @klass = klass
103
+ @args = args
104
+ end
105
+
106
+ def make_new
107
+ @klass.new(*@args)
108
+ end
109
+ end
110
+ end
111
+ end