shoryuken 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,23 @@
1
+
2
+ module Shoryuken
3
+ class Client
4
+ @@queues = {}
5
+
6
+ def self.queues(name)
7
+ @@queues[name.to_s] ||= sqs.queues.named(name)
8
+ end
9
+
10
+ def self.receive_message(queue, options = {})
11
+ queues(queue).receive_message(Hash(options))
12
+ end
13
+
14
+ def self.reset!
15
+ # for test purposes
16
+ @@queues = {}
17
+ end
18
+
19
+ def self.sqs
20
+ @sqs ||= AWS::SQS.new
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,47 @@
1
+ begin
2
+ require 'active_support/core_ext/hash/keys'
3
+ require 'active_support/core_ext/hash/deep_merge'
4
+ rescue LoadError
5
+ class Hash
6
+ def stringify_keys
7
+ keys.each do |key|
8
+ self[key.to_s] = delete(key)
9
+ end
10
+ self
11
+ end if !{}.respond_to?(:stringify_keys)
12
+
13
+ def symbolize_keys
14
+ keys.each do |key|
15
+ self[(key.to_sym rescue key) || key] = delete(key)
16
+ end
17
+ self
18
+ end if !{}.respond_to?(:symbolize_keys)
19
+
20
+ def deep_symbolize_keys
21
+ keys.each do |key|
22
+ value = delete(key)
23
+ self[(key.to_sym rescue key) || key] = value
24
+
25
+ value.deep_symbolize_keys if value.is_a? Hash
26
+ end
27
+ self
28
+ end if !{}.respond_to?(:deep_symbolize_keys)
29
+ end
30
+ end
31
+
32
+ begin
33
+ require 'active_support/core_ext/string/inflections'
34
+ rescue LoadError
35
+ class String
36
+ def constantize
37
+ names = self.split('::')
38
+ names.shift if names.empty? || names.first.empty?
39
+
40
+ constant = Object
41
+ names.each do |name|
42
+ constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
43
+ end
44
+ constant
45
+ end
46
+ end if !"".respond_to?(:constantize)
47
+ end
@@ -0,0 +1,49 @@
1
+ module Shoryuken
2
+ class Fetcher
3
+ include Celluloid
4
+ include Util
5
+
6
+ def initialize(manager)
7
+ @manager = manager
8
+ end
9
+
10
+ def receive_message(queue, limit)
11
+ # AWS limits the batch size by 10
12
+ limit = limit > 10 ? 10 : limit
13
+
14
+ Shoryuken::Client.receive_message queue, Shoryuken.options[:aws][:receive_message].to_h.merge(limit: limit)
15
+ end
16
+
17
+ def fetch(queue, available_processors)
18
+ watchdog('Fetcher#fetch died') do
19
+ started_at = Time.now
20
+
21
+ logger.info "Looking for new messages queue '#{queue}'"
22
+
23
+ begin
24
+ if (sqs_msgs = Array(receive_message(queue, available_processors))).any?
25
+ logger.info "Message found for queue '#{queue}'"
26
+
27
+ sqs_msgs.each do |sqs_msg|
28
+ @manager.async.rebalance_queue_weight!(queue)
29
+ @manager.async.assign(queue, sqs_msg)
30
+ end
31
+ else
32
+ logger.info "No message found for queue '#{queue}'"
33
+
34
+ @manager.async.pause_queue!(queue)
35
+ end
36
+
37
+ @manager.async.dispatch
38
+
39
+ logger.debug "Fetcher#fetch('#{queue}') completed in #{elapsed(started_at)} ms"
40
+ rescue => ex
41
+ logger.error("Error fetching message: #{ex}")
42
+ logger.error(ex.backtrace.first)
43
+
44
+ @manager.async.dispatch
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,41 @@
1
+ module Shoryuken
2
+ class Launcher
3
+ include Celluloid
4
+ include Util
5
+
6
+ trap_exit :actor_died
7
+
8
+ attr_accessor :manager
9
+
10
+ def initialize
11
+ @manager = Shoryuken::Manager.new_link
12
+ @fetcher = Shoryuken::Fetcher.new_link(manager)
13
+
14
+ @done = false
15
+
16
+ manager.fetcher = @fetcher
17
+ end
18
+
19
+ def stop(options = {})
20
+ watchdog('Launcher#stop') do
21
+ @done = true
22
+ @fetcher.terminate if @fetcher.alive?
23
+
24
+ manager.async.stop(shutdown: !!options[:shutdown], timeout: Shoryuken.options[:timeout])
25
+ manager.wait(:shutdown)
26
+ end
27
+ end
28
+
29
+ def run
30
+ watchdog('Launcher#run') do
31
+ manager.async.start
32
+ end
33
+ end
34
+
35
+ def actor_died(actor, reason)
36
+ return if @done
37
+ Shoryuken.logger.warn 'Shoryuken died due to the following error, cannot recover, process exiting'
38
+ exit 1
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,47 @@
1
+ require 'time'
2
+ require 'logger'
3
+
4
+ module Shoryuken
5
+ module Logging
6
+
7
+ class Pretty < Logger::Formatter
8
+ # Provide a call() method that returns the formatted message.
9
+ def call(severity, time, program_name, message)
10
+ "#{time.utc.iso8601} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
11
+ end
12
+
13
+ def context
14
+ c = Thread.current[:shoryuken_context]
15
+ c ? " #{c}" : ''
16
+ end
17
+ end
18
+
19
+ def self.with_context(msg)
20
+ begin
21
+ Thread.current[:shoryuken_context] = msg
22
+ yield
23
+ ensure
24
+ Thread.current[:shoryuken_context] = nil
25
+ end
26
+ end
27
+
28
+ def self.initialize_logger(log_target = STDOUT)
29
+ @logger = Logger.new(log_target)
30
+ @logger.level = Logger::INFO
31
+ @logger.formatter = Pretty.new
32
+ @logger
33
+ end
34
+
35
+ def self.logger
36
+ @logger || initialize_logger
37
+ end
38
+
39
+ def self.logger=(log)
40
+ @logger = (log ? log : Logger.new('/dev/null'))
41
+ end
42
+
43
+ def logger
44
+ shoryuken::Logging.logger
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,220 @@
1
+ require 'yaml'
2
+ require 'aws-sdk'
3
+ require 'celluloid'
4
+
5
+ require 'shoryuken/version'
6
+ require 'shoryuken/manager'
7
+ require 'shoryuken/processor'
8
+ require 'shoryuken/fetcher'
9
+
10
+ module Shoryuken
11
+ class Manager
12
+ include Celluloid
13
+ include Util
14
+
15
+ attr_accessor :fetcher
16
+
17
+ trap_exit :processor_died
18
+
19
+ def initialize
20
+ @count = Shoryuken.options[:concurrency] || 25
21
+ @queues = Shoryuken.queues.dup.uniq
22
+
23
+ @done = false
24
+
25
+ @busy = []
26
+ @ready = @count.times.map { Processor.new_link(current_actor) }
27
+ end
28
+
29
+ def start
30
+ logger.info 'Starting'
31
+
32
+ dispatch
33
+ end
34
+
35
+ def stop(options = {})
36
+ watchdog('Manager#stop died') do
37
+ @done = true
38
+
39
+ @fetcher.terminate if @fetcher.alive?
40
+
41
+ logger.info { "Shutting down #{@ready.size} quiet workers" }
42
+
43
+ @ready.each do |processor|
44
+ processor.terminate if processor.alive?
45
+ end
46
+ @ready.clear
47
+
48
+ return after(0) { signal(:shutdown) } if @busy.empty?
49
+
50
+ if options[:shutdown]
51
+ hard_shutdown_in(options[:timeout])
52
+ else
53
+ soft_shutdown(options[:timeout])
54
+ end
55
+ end
56
+ end
57
+
58
+ def processor_done(queue, processor)
59
+ watchdog('Manager#processor_done died') do
60
+ logger.info "Process done for queue '#{queue}'"
61
+
62
+ @busy.delete processor
63
+
64
+ if stopped?
65
+ processor.terminate if processor.alive?
66
+ else
67
+ @ready << processor
68
+ end
69
+ end
70
+ end
71
+
72
+ def processor_died(processor, reason)
73
+ watchdog("Manager#processor_died died") do
74
+ logger.info "Process died, reason: #{reason}"
75
+
76
+ @busy.delete processor
77
+
78
+ unless stopped?
79
+ @ready << Processor.new_link(current_actor)
80
+ end
81
+ end
82
+ end
83
+
84
+ def stopped?
85
+ @done
86
+ end
87
+
88
+ def assign(queue, sqs_msg)
89
+ watchdog("Manager#assign died") do
90
+ logger.info "Assigning #{sqs_msg.id}"
91
+
92
+ processor = @ready.pop
93
+ @busy << processor
94
+
95
+ processor.async.process(queue, sqs_msg)
96
+ end
97
+ end
98
+
99
+ def rebalance_queue_weight!(queue)
100
+ watchdog('Manager#rebalance_queue_weight! died') do
101
+ if (original = original_queue_weight(queue)) > (current = current_queue_weight(queue))
102
+ if current + 1 > original
103
+ logger.info "Increasing queue '#{queue}' weight to #{current + 1}, original: #{original}"
104
+ else
105
+ logger.info "Queue '#{queue}' is back to its normal weight #{original}"
106
+ end
107
+
108
+ @queues << queue
109
+ end
110
+ end
111
+ end
112
+
113
+ def pause_queue!(queue)
114
+ return unless @queues.include? queue
115
+
116
+ logger.info "Pausing queue '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because the queue is empty"
117
+
118
+ @queues.delete(queue)
119
+
120
+ after(Shoryuken.options[:delay].to_f) { async.restart_queue!(queue) }
121
+ end
122
+
123
+
124
+ def dispatch
125
+ return if stopped?
126
+
127
+ logger.debug { "Ready size: #{@ready.size}" }
128
+ logger.debug { "Busy size: #{@busy.size}" }
129
+ logger.debug { "Queues: #{@queues.inspect}" }
130
+
131
+ if @ready.empty?
132
+ logger.debug { 'Pausing fetcher, because all processors are busy' }
133
+
134
+ after(1) { dispatch }
135
+
136
+ return
137
+ end
138
+
139
+ if queue = next_queue
140
+ @fetcher.async.fetch(queue, @ready.size)
141
+ else
142
+ logger.debug { 'Pausing fetcher, because all queues are paused' }
143
+
144
+ @fetcher_paused = true
145
+ end
146
+ end
147
+
148
+ private
149
+
150
+ def restart_queue!(queue)
151
+ return if stopped?
152
+
153
+ unless @queues.include? queue
154
+ logger.info "Restarting queue '#{queue}'"
155
+
156
+ @queues << queue
157
+
158
+ if @fetcher_paused
159
+ logger.debug { 'Restarting fetcher' }
160
+
161
+ @fetcher_paused = false
162
+
163
+
164
+ dispatch
165
+ end
166
+ end
167
+ end
168
+
169
+ def current_queue_weight(queue)
170
+ queue_weight(@queues, queue)
171
+ end
172
+
173
+ def original_queue_weight(queue)
174
+ queue_weight(Shoryuken.queues, queue)
175
+ end
176
+
177
+ def queue_weight(queues, queue)
178
+ queues.count { |q| q == queue }
179
+ end
180
+
181
+ def next_queue
182
+ return nil if @queues.empty?
183
+
184
+ queue = @queues.shift
185
+ @queues << queue
186
+
187
+ queue
188
+ end
189
+
190
+ def soft_shutdown(delay)
191
+ logger.info { "Waiting for #{@busy.size} busy workers" }
192
+
193
+ if @busy.size > 0
194
+ after(delay) { soft_shutdown(delay) }
195
+ else
196
+ after(0) { signal(:shutdown) }
197
+ end
198
+ end
199
+
200
+ def hard_shutdown_in(delay)
201
+ logger.info { "Waiting for #{@busy.size} busy workers" }
202
+ logger.info { "Pausing up to #{delay} seconds to allow workers to finish..." }
203
+
204
+ after(delay) do
205
+ watchdog("Manager#hard_shutdown_in died") do
206
+ if @busy.size > 0
207
+ logger.info { "Hard shutting down #{@busy.size} busy workers" }
208
+
209
+ @busy.each do |processor|
210
+ t = processor.bare_object.actual_work_thread
211
+ t.raise Shutdown if processor.alive?
212
+ end
213
+ end
214
+
215
+ after(0) { signal(:shutdown) }
216
+ end
217
+ end
218
+ end
219
+ end
220
+ end
@@ -0,0 +1,111 @@
1
+ module Shoryuken
2
+ # Middleware is code configured to run before/after
3
+ # a message is processed. It is patterned after Rack
4
+ # middleware. Middleware exists for the server
5
+ # side (when jobs are actually processed).
6
+ #
7
+ # To modify middleware for the server, just call
8
+ # with another block:
9
+ #
10
+ # Shoryuken.configure_server do |config|
11
+ # config.server_middleware do |chain|
12
+ # chain.add MyServerHook
13
+ # chain.remove ActiveRecord
14
+ # end
15
+ # end
16
+ #
17
+ # To insert immediately preceding another entry:
18
+ #
19
+ # Shoryuken.configure_server do |config|
20
+ # config.server_middleware do |chain|
21
+ # chain.insert_before ActiveRecord, MyServerHook
22
+ # end
23
+ # end
24
+ #
25
+ # To insert immediately after another entry:
26
+ #
27
+ # Shoryuken.configure_server do |config|
28
+ # config.server_middleware do |chain|
29
+ # chain.insert_after ActiveRecord, MyServerHook
30
+ # end
31
+ # end
32
+ #
33
+ # This is an example of a minimal server middleware:
34
+ #
35
+ # class MyServerHook
36
+ # def call(worker_instance, queue, sqs_msg)
37
+ # puts 'Before work'
38
+ # yield
39
+ # puts 'After work'
40
+ # end
41
+ # end
42
+ #
43
+ module Middleware
44
+ class Chain
45
+ attr_reader :entries
46
+
47
+ def initialize
48
+ @entries = []
49
+ yield self if block_given?
50
+ end
51
+
52
+ def remove(klass)
53
+ entries.delete_if { |entry| entry.klass == klass }
54
+ end
55
+
56
+ def add(klass, *args)
57
+ entries << Entry.new(klass, *args) unless exists?(klass)
58
+ end
59
+
60
+ def insert_before(oldklass, newklass, *args)
61
+ i = entries.index { |entry| entry.klass == newklass }
62
+ new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
63
+ i = entries.find_index { |entry| entry.klass == oldklass } || 0
64
+ entries.insert(i, new_entry)
65
+ end
66
+
67
+ def insert_after(oldklass, newklass, *args)
68
+ i = entries.index { |entry| entry.klass == newklass }
69
+ new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
70
+ i = entries.find_index { |entry| entry.klass == oldklass } || entries.count - 1
71
+ entries.insert(i+1, new_entry)
72
+ end
73
+
74
+ def exists?(klass)
75
+ entries.any? { |entry| entry.klass == klass }
76
+ end
77
+
78
+ def retrieve
79
+ entries.map(&:make_new)
80
+ end
81
+
82
+ def clear
83
+ entries.clear
84
+ end
85
+
86
+ def invoke(*args, &final_action)
87
+ chain = retrieve.dup
88
+ traverse_chain = lambda do
89
+ if chain.empty?
90
+ final_action.call
91
+ else
92
+ chain.shift.call(*args, &traverse_chain)
93
+ end
94
+ end
95
+ traverse_chain.call
96
+ end
97
+ end
98
+
99
+ class Entry
100
+ attr_reader :klass
101
+ def initialize(klass, *args)
102
+ @klass = klass
103
+ @args = args
104
+ end
105
+
106
+ def make_new
107
+ @klass.new(*@args)
108
+ end
109
+ end
110
+ end
111
+ end