shoryuken 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +25 -0
- data/.rspec +1 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +5 -0
- data/README.md +135 -0
- data/Rakefile +41 -0
- data/bin/shoryuken +13 -0
- data/examples/all.rb +5 -0
- data/examples/shoryuken_worker.rb +11 -0
- data/examples/sidekiq_worker.rb +11 -0
- data/examples/uppercut_worker.rb +11 -0
- data/lib/shoryuken.rb +69 -0
- data/lib/shoryuken/cli.rb +222 -0
- data/lib/shoryuken/client.rb +23 -0
- data/lib/shoryuken/core_ext.rb +47 -0
- data/lib/shoryuken/fetcher.rb +49 -0
- data/lib/shoryuken/launcher.rb +41 -0
- data/lib/shoryuken/logging.rb +47 -0
- data/lib/shoryuken/manager.rb +220 -0
- data/lib/shoryuken/middleware/chain.rb +111 -0
- data/lib/shoryuken/middleware/server/auto_delete.rb +14 -0
- data/lib/shoryuken/middleware/server/logging.rb +23 -0
- data/lib/shoryuken/processor.rb +36 -0
- data/lib/shoryuken/util.rb +19 -0
- data/lib/shoryuken/version.rb +3 -0
- data/lib/shoryuken/worker.rb +29 -0
- data/shoryuken.gemspec +28 -0
- data/shoryuken.jpg +0 -0
- data/spec/shoryuken/chain_spec.rb +48 -0
- data/spec/shoryuken/client_spec.rb +22 -0
- data/spec/shoryuken/core_ext_spec.rb +12 -0
- data/spec/shoryuken/fetcher_spec.rb +36 -0
- data/spec/shoryuken/integration/launcher_spec.rb +36 -0
- data/spec/shoryuken/manager_spec.rb +73 -0
- data/spec/shoryuken/processor_spec.rb +93 -0
- data/spec/shoryuken/worker_spec.rb +28 -0
- data/spec/spec_helper.rb +40 -0
- metadata +189 -0
@@ -0,0 +1,23 @@
|
|
1
|
+
|
2
|
+
module Shoryuken
|
3
|
+
class Client
|
4
|
+
@@queues = {}
|
5
|
+
|
6
|
+
def self.queues(name)
|
7
|
+
@@queues[name.to_s] ||= sqs.queues.named(name)
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.receive_message(queue, options = {})
|
11
|
+
queues(queue).receive_message(Hash(options))
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.reset!
|
15
|
+
# for test purposes
|
16
|
+
@@queues = {}
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.sqs
|
20
|
+
@sqs ||= AWS::SQS.new
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
begin
|
2
|
+
require 'active_support/core_ext/hash/keys'
|
3
|
+
require 'active_support/core_ext/hash/deep_merge'
|
4
|
+
rescue LoadError
|
5
|
+
class Hash
|
6
|
+
def stringify_keys
|
7
|
+
keys.each do |key|
|
8
|
+
self[key.to_s] = delete(key)
|
9
|
+
end
|
10
|
+
self
|
11
|
+
end if !{}.respond_to?(:stringify_keys)
|
12
|
+
|
13
|
+
def symbolize_keys
|
14
|
+
keys.each do |key|
|
15
|
+
self[(key.to_sym rescue key) || key] = delete(key)
|
16
|
+
end
|
17
|
+
self
|
18
|
+
end if !{}.respond_to?(:symbolize_keys)
|
19
|
+
|
20
|
+
def deep_symbolize_keys
|
21
|
+
keys.each do |key|
|
22
|
+
value = delete(key)
|
23
|
+
self[(key.to_sym rescue key) || key] = value
|
24
|
+
|
25
|
+
value.deep_symbolize_keys if value.is_a? Hash
|
26
|
+
end
|
27
|
+
self
|
28
|
+
end if !{}.respond_to?(:deep_symbolize_keys)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
begin
|
33
|
+
require 'active_support/core_ext/string/inflections'
|
34
|
+
rescue LoadError
|
35
|
+
class String
|
36
|
+
def constantize
|
37
|
+
names = self.split('::')
|
38
|
+
names.shift if names.empty? || names.first.empty?
|
39
|
+
|
40
|
+
constant = Object
|
41
|
+
names.each do |name|
|
42
|
+
constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
|
43
|
+
end
|
44
|
+
constant
|
45
|
+
end
|
46
|
+
end if !"".respond_to?(:constantize)
|
47
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
class Fetcher
|
3
|
+
include Celluloid
|
4
|
+
include Util
|
5
|
+
|
6
|
+
def initialize(manager)
|
7
|
+
@manager = manager
|
8
|
+
end
|
9
|
+
|
10
|
+
def receive_message(queue, limit)
|
11
|
+
# AWS limits the batch size by 10
|
12
|
+
limit = limit > 10 ? 10 : limit
|
13
|
+
|
14
|
+
Shoryuken::Client.receive_message queue, Shoryuken.options[:aws][:receive_message].to_h.merge(limit: limit)
|
15
|
+
end
|
16
|
+
|
17
|
+
def fetch(queue, available_processors)
|
18
|
+
watchdog('Fetcher#fetch died') do
|
19
|
+
started_at = Time.now
|
20
|
+
|
21
|
+
logger.info "Looking for new messages queue '#{queue}'"
|
22
|
+
|
23
|
+
begin
|
24
|
+
if (sqs_msgs = Array(receive_message(queue, available_processors))).any?
|
25
|
+
logger.info "Message found for queue '#{queue}'"
|
26
|
+
|
27
|
+
sqs_msgs.each do |sqs_msg|
|
28
|
+
@manager.async.rebalance_queue_weight!(queue)
|
29
|
+
@manager.async.assign(queue, sqs_msg)
|
30
|
+
end
|
31
|
+
else
|
32
|
+
logger.info "No message found for queue '#{queue}'"
|
33
|
+
|
34
|
+
@manager.async.pause_queue!(queue)
|
35
|
+
end
|
36
|
+
|
37
|
+
@manager.async.dispatch
|
38
|
+
|
39
|
+
logger.debug "Fetcher#fetch('#{queue}') completed in #{elapsed(started_at)} ms"
|
40
|
+
rescue => ex
|
41
|
+
logger.error("Error fetching message: #{ex}")
|
42
|
+
logger.error(ex.backtrace.first)
|
43
|
+
|
44
|
+
@manager.async.dispatch
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
class Launcher
|
3
|
+
include Celluloid
|
4
|
+
include Util
|
5
|
+
|
6
|
+
trap_exit :actor_died
|
7
|
+
|
8
|
+
attr_accessor :manager
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@manager = Shoryuken::Manager.new_link
|
12
|
+
@fetcher = Shoryuken::Fetcher.new_link(manager)
|
13
|
+
|
14
|
+
@done = false
|
15
|
+
|
16
|
+
manager.fetcher = @fetcher
|
17
|
+
end
|
18
|
+
|
19
|
+
def stop(options = {})
|
20
|
+
watchdog('Launcher#stop') do
|
21
|
+
@done = true
|
22
|
+
@fetcher.terminate if @fetcher.alive?
|
23
|
+
|
24
|
+
manager.async.stop(shutdown: !!options[:shutdown], timeout: Shoryuken.options[:timeout])
|
25
|
+
manager.wait(:shutdown)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def run
|
30
|
+
watchdog('Launcher#run') do
|
31
|
+
manager.async.start
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def actor_died(actor, reason)
|
36
|
+
return if @done
|
37
|
+
Shoryuken.logger.warn 'Shoryuken died due to the following error, cannot recover, process exiting'
|
38
|
+
exit 1
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'time'
|
2
|
+
require 'logger'
|
3
|
+
|
4
|
+
module Shoryuken
|
5
|
+
module Logging
|
6
|
+
|
7
|
+
class Pretty < Logger::Formatter
|
8
|
+
# Provide a call() method that returns the formatted message.
|
9
|
+
def call(severity, time, program_name, message)
|
10
|
+
"#{time.utc.iso8601} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
|
11
|
+
end
|
12
|
+
|
13
|
+
def context
|
14
|
+
c = Thread.current[:shoryuken_context]
|
15
|
+
c ? " #{c}" : ''
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.with_context(msg)
|
20
|
+
begin
|
21
|
+
Thread.current[:shoryuken_context] = msg
|
22
|
+
yield
|
23
|
+
ensure
|
24
|
+
Thread.current[:shoryuken_context] = nil
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.initialize_logger(log_target = STDOUT)
|
29
|
+
@logger = Logger.new(log_target)
|
30
|
+
@logger.level = Logger::INFO
|
31
|
+
@logger.formatter = Pretty.new
|
32
|
+
@logger
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.logger
|
36
|
+
@logger || initialize_logger
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.logger=(log)
|
40
|
+
@logger = (log ? log : Logger.new('/dev/null'))
|
41
|
+
end
|
42
|
+
|
43
|
+
def logger
|
44
|
+
shoryuken::Logging.logger
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,220 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'aws-sdk'
|
3
|
+
require 'celluloid'
|
4
|
+
|
5
|
+
require 'shoryuken/version'
|
6
|
+
require 'shoryuken/manager'
|
7
|
+
require 'shoryuken/processor'
|
8
|
+
require 'shoryuken/fetcher'
|
9
|
+
|
10
|
+
module Shoryuken
|
11
|
+
class Manager
|
12
|
+
include Celluloid
|
13
|
+
include Util
|
14
|
+
|
15
|
+
attr_accessor :fetcher
|
16
|
+
|
17
|
+
trap_exit :processor_died
|
18
|
+
|
19
|
+
def initialize
|
20
|
+
@count = Shoryuken.options[:concurrency] || 25
|
21
|
+
@queues = Shoryuken.queues.dup.uniq
|
22
|
+
|
23
|
+
@done = false
|
24
|
+
|
25
|
+
@busy = []
|
26
|
+
@ready = @count.times.map { Processor.new_link(current_actor) }
|
27
|
+
end
|
28
|
+
|
29
|
+
def start
|
30
|
+
logger.info 'Starting'
|
31
|
+
|
32
|
+
dispatch
|
33
|
+
end
|
34
|
+
|
35
|
+
def stop(options = {})
|
36
|
+
watchdog('Manager#stop died') do
|
37
|
+
@done = true
|
38
|
+
|
39
|
+
@fetcher.terminate if @fetcher.alive?
|
40
|
+
|
41
|
+
logger.info { "Shutting down #{@ready.size} quiet workers" }
|
42
|
+
|
43
|
+
@ready.each do |processor|
|
44
|
+
processor.terminate if processor.alive?
|
45
|
+
end
|
46
|
+
@ready.clear
|
47
|
+
|
48
|
+
return after(0) { signal(:shutdown) } if @busy.empty?
|
49
|
+
|
50
|
+
if options[:shutdown]
|
51
|
+
hard_shutdown_in(options[:timeout])
|
52
|
+
else
|
53
|
+
soft_shutdown(options[:timeout])
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def processor_done(queue, processor)
|
59
|
+
watchdog('Manager#processor_done died') do
|
60
|
+
logger.info "Process done for queue '#{queue}'"
|
61
|
+
|
62
|
+
@busy.delete processor
|
63
|
+
|
64
|
+
if stopped?
|
65
|
+
processor.terminate if processor.alive?
|
66
|
+
else
|
67
|
+
@ready << processor
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def processor_died(processor, reason)
|
73
|
+
watchdog("Manager#processor_died died") do
|
74
|
+
logger.info "Process died, reason: #{reason}"
|
75
|
+
|
76
|
+
@busy.delete processor
|
77
|
+
|
78
|
+
unless stopped?
|
79
|
+
@ready << Processor.new_link(current_actor)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def stopped?
|
85
|
+
@done
|
86
|
+
end
|
87
|
+
|
88
|
+
def assign(queue, sqs_msg)
|
89
|
+
watchdog("Manager#assign died") do
|
90
|
+
logger.info "Assigning #{sqs_msg.id}"
|
91
|
+
|
92
|
+
processor = @ready.pop
|
93
|
+
@busy << processor
|
94
|
+
|
95
|
+
processor.async.process(queue, sqs_msg)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
def rebalance_queue_weight!(queue)
|
100
|
+
watchdog('Manager#rebalance_queue_weight! died') do
|
101
|
+
if (original = original_queue_weight(queue)) > (current = current_queue_weight(queue))
|
102
|
+
if current + 1 > original
|
103
|
+
logger.info "Increasing queue '#{queue}' weight to #{current + 1}, original: #{original}"
|
104
|
+
else
|
105
|
+
logger.info "Queue '#{queue}' is back to its normal weight #{original}"
|
106
|
+
end
|
107
|
+
|
108
|
+
@queues << queue
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def pause_queue!(queue)
|
114
|
+
return unless @queues.include? queue
|
115
|
+
|
116
|
+
logger.info "Pausing queue '#{queue}' for #{Shoryuken.options[:delay].to_f} seconds, because the queue is empty"
|
117
|
+
|
118
|
+
@queues.delete(queue)
|
119
|
+
|
120
|
+
after(Shoryuken.options[:delay].to_f) { async.restart_queue!(queue) }
|
121
|
+
end
|
122
|
+
|
123
|
+
|
124
|
+
def dispatch
|
125
|
+
return if stopped?
|
126
|
+
|
127
|
+
logger.debug { "Ready size: #{@ready.size}" }
|
128
|
+
logger.debug { "Busy size: #{@busy.size}" }
|
129
|
+
logger.debug { "Queues: #{@queues.inspect}" }
|
130
|
+
|
131
|
+
if @ready.empty?
|
132
|
+
logger.debug { 'Pausing fetcher, because all processors are busy' }
|
133
|
+
|
134
|
+
after(1) { dispatch }
|
135
|
+
|
136
|
+
return
|
137
|
+
end
|
138
|
+
|
139
|
+
if queue = next_queue
|
140
|
+
@fetcher.async.fetch(queue, @ready.size)
|
141
|
+
else
|
142
|
+
logger.debug { 'Pausing fetcher, because all queues are paused' }
|
143
|
+
|
144
|
+
@fetcher_paused = true
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
private
|
149
|
+
|
150
|
+
def restart_queue!(queue)
|
151
|
+
return if stopped?
|
152
|
+
|
153
|
+
unless @queues.include? queue
|
154
|
+
logger.info "Restarting queue '#{queue}'"
|
155
|
+
|
156
|
+
@queues << queue
|
157
|
+
|
158
|
+
if @fetcher_paused
|
159
|
+
logger.debug { 'Restarting fetcher' }
|
160
|
+
|
161
|
+
@fetcher_paused = false
|
162
|
+
|
163
|
+
|
164
|
+
dispatch
|
165
|
+
end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
def current_queue_weight(queue)
|
170
|
+
queue_weight(@queues, queue)
|
171
|
+
end
|
172
|
+
|
173
|
+
def original_queue_weight(queue)
|
174
|
+
queue_weight(Shoryuken.queues, queue)
|
175
|
+
end
|
176
|
+
|
177
|
+
def queue_weight(queues, queue)
|
178
|
+
queues.count { |q| q == queue }
|
179
|
+
end
|
180
|
+
|
181
|
+
def next_queue
|
182
|
+
return nil if @queues.empty?
|
183
|
+
|
184
|
+
queue = @queues.shift
|
185
|
+
@queues << queue
|
186
|
+
|
187
|
+
queue
|
188
|
+
end
|
189
|
+
|
190
|
+
def soft_shutdown(delay)
|
191
|
+
logger.info { "Waiting for #{@busy.size} busy workers" }
|
192
|
+
|
193
|
+
if @busy.size > 0
|
194
|
+
after(delay) { soft_shutdown(delay) }
|
195
|
+
else
|
196
|
+
after(0) { signal(:shutdown) }
|
197
|
+
end
|
198
|
+
end
|
199
|
+
|
200
|
+
def hard_shutdown_in(delay)
|
201
|
+
logger.info { "Waiting for #{@busy.size} busy workers" }
|
202
|
+
logger.info { "Pausing up to #{delay} seconds to allow workers to finish..." }
|
203
|
+
|
204
|
+
after(delay) do
|
205
|
+
watchdog("Manager#hard_shutdown_in died") do
|
206
|
+
if @busy.size > 0
|
207
|
+
logger.info { "Hard shutting down #{@busy.size} busy workers" }
|
208
|
+
|
209
|
+
@busy.each do |processor|
|
210
|
+
t = processor.bare_object.actual_work_thread
|
211
|
+
t.raise Shutdown if processor.alive?
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
after(0) { signal(:shutdown) }
|
216
|
+
end
|
217
|
+
end
|
218
|
+
end
|
219
|
+
end
|
220
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
module Shoryuken
|
2
|
+
# Middleware is code configured to run before/after
|
3
|
+
# a message is processed. It is patterned after Rack
|
4
|
+
# middleware. Middleware exists for the server
|
5
|
+
# side (when jobs are actually processed).
|
6
|
+
#
|
7
|
+
# To modify middleware for the server, just call
|
8
|
+
# with another block:
|
9
|
+
#
|
10
|
+
# Shoryuken.configure_server do |config|
|
11
|
+
# config.server_middleware do |chain|
|
12
|
+
# chain.add MyServerHook
|
13
|
+
# chain.remove ActiveRecord
|
14
|
+
# end
|
15
|
+
# end
|
16
|
+
#
|
17
|
+
# To insert immediately preceding another entry:
|
18
|
+
#
|
19
|
+
# Shoryuken.configure_server do |config|
|
20
|
+
# config.server_middleware do |chain|
|
21
|
+
# chain.insert_before ActiveRecord, MyServerHook
|
22
|
+
# end
|
23
|
+
# end
|
24
|
+
#
|
25
|
+
# To insert immediately after another entry:
|
26
|
+
#
|
27
|
+
# Shoryuken.configure_server do |config|
|
28
|
+
# config.server_middleware do |chain|
|
29
|
+
# chain.insert_after ActiveRecord, MyServerHook
|
30
|
+
# end
|
31
|
+
# end
|
32
|
+
#
|
33
|
+
# This is an example of a minimal server middleware:
|
34
|
+
#
|
35
|
+
# class MyServerHook
|
36
|
+
# def call(worker_instance, queue, sqs_msg)
|
37
|
+
# puts 'Before work'
|
38
|
+
# yield
|
39
|
+
# puts 'After work'
|
40
|
+
# end
|
41
|
+
# end
|
42
|
+
#
|
43
|
+
module Middleware
|
44
|
+
class Chain
|
45
|
+
attr_reader :entries
|
46
|
+
|
47
|
+
def initialize
|
48
|
+
@entries = []
|
49
|
+
yield self if block_given?
|
50
|
+
end
|
51
|
+
|
52
|
+
def remove(klass)
|
53
|
+
entries.delete_if { |entry| entry.klass == klass }
|
54
|
+
end
|
55
|
+
|
56
|
+
def add(klass, *args)
|
57
|
+
entries << Entry.new(klass, *args) unless exists?(klass)
|
58
|
+
end
|
59
|
+
|
60
|
+
def insert_before(oldklass, newklass, *args)
|
61
|
+
i = entries.index { |entry| entry.klass == newklass }
|
62
|
+
new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
|
63
|
+
i = entries.find_index { |entry| entry.klass == oldklass } || 0
|
64
|
+
entries.insert(i, new_entry)
|
65
|
+
end
|
66
|
+
|
67
|
+
def insert_after(oldklass, newklass, *args)
|
68
|
+
i = entries.index { |entry| entry.klass == newklass }
|
69
|
+
new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
|
70
|
+
i = entries.find_index { |entry| entry.klass == oldklass } || entries.count - 1
|
71
|
+
entries.insert(i+1, new_entry)
|
72
|
+
end
|
73
|
+
|
74
|
+
def exists?(klass)
|
75
|
+
entries.any? { |entry| entry.klass == klass }
|
76
|
+
end
|
77
|
+
|
78
|
+
def retrieve
|
79
|
+
entries.map(&:make_new)
|
80
|
+
end
|
81
|
+
|
82
|
+
def clear
|
83
|
+
entries.clear
|
84
|
+
end
|
85
|
+
|
86
|
+
def invoke(*args, &final_action)
|
87
|
+
chain = retrieve.dup
|
88
|
+
traverse_chain = lambda do
|
89
|
+
if chain.empty?
|
90
|
+
final_action.call
|
91
|
+
else
|
92
|
+
chain.shift.call(*args, &traverse_chain)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
traverse_chain.call
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
class Entry
|
100
|
+
attr_reader :klass
|
101
|
+
def initialize(klass, *args)
|
102
|
+
@klass = klass
|
103
|
+
@args = args
|
104
|
+
end
|
105
|
+
|
106
|
+
def make_new
|
107
|
+
@klass.new(*@args)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|