toiler 0.3.6 → 0.4.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +35 -35
- data/.gitmodules +4 -4
- data/.ruby-version +1 -1
- data/Gemfile +9 -9
- data/Gemfile.lock +41 -41
- data/LICENSE +6 -6
- data/README.md +125 -125
- data/Rakefile +1 -1
- data/bin/toiler +12 -12
- data/lib/toiler.rb +55 -55
- data/lib/toiler/actor/fetcher.rb +116 -116
- data/lib/toiler/actor/processor.rb +124 -123
- data/lib/toiler/actor/supervisor.rb +55 -55
- data/lib/toiler/actor/utils/actor_logging.rb +28 -28
- data/lib/toiler/aws/message.rb +64 -64
- data/lib/toiler/aws/queue.rb +61 -61
- data/lib/toiler/cli.rb +164 -164
- data/lib/toiler/utils/argument_parser.rb +50 -50
- data/lib/toiler/utils/environment_loader.rb +104 -104
- data/lib/toiler/utils/logging.rb +41 -41
- data/lib/toiler/version.rb +4 -4
- data/lib/toiler/worker.rb +62 -62
- data/toiler.gemspec +30 -30
- metadata +4 -4
@@ -1,55 +1,55 @@
|
|
1
|
-
require 'toiler/actor/fetcher'
|
2
|
-
require 'toiler/actor/processor'
|
3
|
-
|
4
|
-
module Toiler
|
5
|
-
module Actor
|
6
|
-
# Actor that starts and supervises Toiler's actors
|
7
|
-
class Supervisor < Concurrent::Actor::RestartingContext
|
8
|
-
include Utils::ActorLogging
|
9
|
-
|
10
|
-
attr_accessor :client
|
11
|
-
|
12
|
-
def initialize
|
13
|
-
@client = ::Aws::SQS::Client.new
|
14
|
-
spawn_fetchers
|
15
|
-
spawn_processors
|
16
|
-
end
|
17
|
-
|
18
|
-
def on_message(_msg)
|
19
|
-
pass
|
20
|
-
end
|
21
|
-
|
22
|
-
def queues
|
23
|
-
Toiler.worker_class_registry
|
24
|
-
end
|
25
|
-
|
26
|
-
def spawn_fetchers
|
27
|
-
queues.each do |queue, _klass|
|
28
|
-
begin
|
29
|
-
fetcher = Actor::Fetcher.spawn! name: "fetcher_#{queue}".to_sym,
|
30
|
-
supervise: true, args: [queue, client]
|
31
|
-
Toiler.set_fetcher queue, fetcher
|
32
|
-
rescue StandardError => e
|
33
|
-
error "Failed to start Fetcher for queue #{queue}: #{e.message}\n#{e.backtrace.join("\n")}"
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
38
|
-
def spawn_processors
|
39
|
-
queues.each do |queue, klass|
|
40
|
-
name = "processor_pool_#{queue}".to_sym
|
41
|
-
count = klass.concurrency
|
42
|
-
begin
|
43
|
-
pool = Concurrent::Actor::Utils::Pool.spawn! name, count do |index|
|
44
|
-
Actor::Processor.spawn name: "processor_#{queue}_#{index}".to_sym,
|
45
|
-
supervise: true, args: [queue]
|
46
|
-
end
|
47
|
-
Toiler.set_processor_pool queue, pool
|
48
|
-
rescue StandardError => e
|
49
|
-
error "Failed to spawn Processor Pool for queue #{queue}: #{e.message}\n#{e.backtrace.join("\n")}"
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
1
|
+
require 'toiler/actor/fetcher'
|
2
|
+
require 'toiler/actor/processor'
|
3
|
+
|
4
|
+
module Toiler
|
5
|
+
module Actor
|
6
|
+
# Actor that starts and supervises Toiler's actors
|
7
|
+
class Supervisor < Concurrent::Actor::RestartingContext
|
8
|
+
include Utils::ActorLogging
|
9
|
+
|
10
|
+
attr_accessor :client
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@client = ::Aws::SQS::Client.new
|
14
|
+
spawn_fetchers
|
15
|
+
spawn_processors
|
16
|
+
end
|
17
|
+
|
18
|
+
def on_message(_msg)
|
19
|
+
pass
|
20
|
+
end
|
21
|
+
|
22
|
+
def queues
|
23
|
+
Toiler.worker_class_registry
|
24
|
+
end
|
25
|
+
|
26
|
+
def spawn_fetchers
|
27
|
+
queues.each do |queue, _klass|
|
28
|
+
begin
|
29
|
+
fetcher = Actor::Fetcher.spawn! name: "fetcher_#{queue}".to_sym,
|
30
|
+
supervise: true, args: [queue, client]
|
31
|
+
Toiler.set_fetcher queue, fetcher
|
32
|
+
rescue StandardError => e
|
33
|
+
error "Failed to start Fetcher for queue #{queue}: #{e.message}\n#{e.backtrace.join("\n")}"
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def spawn_processors
|
39
|
+
queues.each do |queue, klass|
|
40
|
+
name = "processor_pool_#{queue}".to_sym
|
41
|
+
count = klass.concurrency
|
42
|
+
begin
|
43
|
+
pool = Concurrent::Actor::Utils::Pool.spawn! name, count do |index|
|
44
|
+
Actor::Processor.spawn name: "processor_#{queue}_#{index}".to_sym,
|
45
|
+
supervise: true, args: [queue]
|
46
|
+
end
|
47
|
+
Toiler.set_processor_pool queue, pool
|
48
|
+
rescue StandardError => e
|
49
|
+
error "Failed to spawn Processor Pool for queue #{queue}: #{e.message}\n#{e.backtrace.join("\n")}"
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -1,28 +1,28 @@
|
|
1
|
-
module Toiler
|
2
|
-
module Actor
|
3
|
-
module Utils
|
4
|
-
# Provides helper methods for logging
|
5
|
-
module ActorLogging
|
6
|
-
def error(msg)
|
7
|
-
log Logger::Severity::ERROR, msg
|
8
|
-
end
|
9
|
-
|
10
|
-
def info(msg)
|
11
|
-
log Logger::Severity::INFO, msg
|
12
|
-
end
|
13
|
-
|
14
|
-
def debug(msg)
|
15
|
-
log Logger::Severity::DEBUG, msg
|
16
|
-
end
|
17
|
-
|
18
|
-
def warn(msg)
|
19
|
-
log Logger::Severity::WARN, smsg
|
20
|
-
end
|
21
|
-
|
22
|
-
def fatal(msg)
|
23
|
-
log Logger::Severity::FATAL, msg
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
1
|
+
module Toiler
|
2
|
+
module Actor
|
3
|
+
module Utils
|
4
|
+
# Provides helper methods for logging
|
5
|
+
module ActorLogging
|
6
|
+
def error(msg)
|
7
|
+
log Logger::Severity::ERROR, msg
|
8
|
+
end
|
9
|
+
|
10
|
+
def info(msg)
|
11
|
+
log Logger::Severity::INFO, msg
|
12
|
+
end
|
13
|
+
|
14
|
+
def debug(msg)
|
15
|
+
log Logger::Severity::DEBUG, msg
|
16
|
+
end
|
17
|
+
|
18
|
+
def warn(msg)
|
19
|
+
log Logger::Severity::WARN, smsg
|
20
|
+
end
|
21
|
+
|
22
|
+
def fatal(msg)
|
23
|
+
log Logger::Severity::FATAL, msg
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
data/lib/toiler/aws/message.rb
CHANGED
@@ -1,64 +1,64 @@
|
|
1
|
-
module Toiler
|
2
|
-
module Aws
|
3
|
-
# SQS Message abstraction
|
4
|
-
# Provides methods for querying and acting on a SQS message
|
5
|
-
class Message
|
6
|
-
attr_accessor :client, :queue_url, :data
|
7
|
-
|
8
|
-
def initialize(client, queue_url, data)
|
9
|
-
@client = client
|
10
|
-
@queue_url = queue_url
|
11
|
-
@data = data
|
12
|
-
end
|
13
|
-
|
14
|
-
def delete
|
15
|
-
client.delete_message(
|
16
|
-
queue_url: queue_url,
|
17
|
-
receipt_handle: data.receipt_handle
|
18
|
-
)
|
19
|
-
end
|
20
|
-
|
21
|
-
def change_visibility(options)
|
22
|
-
client.change_message_visibility(
|
23
|
-
options.merge(queue_url: queue_url, receipt_handle: receipt_handle)
|
24
|
-
)
|
25
|
-
end
|
26
|
-
|
27
|
-
def visibility_timeout=(timeout)
|
28
|
-
client.change_message_visibility(
|
29
|
-
queue_url: queue_url,
|
30
|
-
receipt_handle: data.receipt_handle,
|
31
|
-
visibility_timeout: timeout
|
32
|
-
)
|
33
|
-
end
|
34
|
-
|
35
|
-
def message_id
|
36
|
-
data.message_id
|
37
|
-
end
|
38
|
-
|
39
|
-
def receipt_handle
|
40
|
-
data.receipt_handle
|
41
|
-
end
|
42
|
-
|
43
|
-
def md5_of_body
|
44
|
-
data.md5_of_body
|
45
|
-
end
|
46
|
-
|
47
|
-
def body
|
48
|
-
data.body
|
49
|
-
end
|
50
|
-
|
51
|
-
def attributes
|
52
|
-
data.attributes
|
53
|
-
end
|
54
|
-
|
55
|
-
def md5_of_message_attributes
|
56
|
-
data.md5_of_message_attributes
|
57
|
-
end
|
58
|
-
|
59
|
-
def message_attributes
|
60
|
-
data.message_attributes
|
61
|
-
end
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|
1
|
+
module Toiler
|
2
|
+
module Aws
|
3
|
+
# SQS Message abstraction
|
4
|
+
# Provides methods for querying and acting on a SQS message
|
5
|
+
class Message
|
6
|
+
attr_accessor :client, :queue_url, :data
|
7
|
+
|
8
|
+
def initialize(client, queue_url, data)
|
9
|
+
@client = client
|
10
|
+
@queue_url = queue_url
|
11
|
+
@data = data
|
12
|
+
end
|
13
|
+
|
14
|
+
def delete
|
15
|
+
client.delete_message(
|
16
|
+
queue_url: queue_url,
|
17
|
+
receipt_handle: data.receipt_handle
|
18
|
+
)
|
19
|
+
end
|
20
|
+
|
21
|
+
def change_visibility(options)
|
22
|
+
client.change_message_visibility(
|
23
|
+
options.merge(queue_url: queue_url, receipt_handle: receipt_handle)
|
24
|
+
)
|
25
|
+
end
|
26
|
+
|
27
|
+
def visibility_timeout=(timeout)
|
28
|
+
client.change_message_visibility(
|
29
|
+
queue_url: queue_url,
|
30
|
+
receipt_handle: data.receipt_handle,
|
31
|
+
visibility_timeout: timeout
|
32
|
+
)
|
33
|
+
end
|
34
|
+
|
35
|
+
def message_id
|
36
|
+
data.message_id
|
37
|
+
end
|
38
|
+
|
39
|
+
def receipt_handle
|
40
|
+
data.receipt_handle
|
41
|
+
end
|
42
|
+
|
43
|
+
def md5_of_body
|
44
|
+
data.md5_of_body
|
45
|
+
end
|
46
|
+
|
47
|
+
def body
|
48
|
+
data.body
|
49
|
+
end
|
50
|
+
|
51
|
+
def attributes
|
52
|
+
data.attributes
|
53
|
+
end
|
54
|
+
|
55
|
+
def md5_of_message_attributes
|
56
|
+
data.md5_of_message_attributes
|
57
|
+
end
|
58
|
+
|
59
|
+
def message_attributes
|
60
|
+
data.message_attributes
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
data/lib/toiler/aws/queue.rb
CHANGED
@@ -1,61 +1,61 @@
|
|
1
|
-
require 'toiler/aws/message'
|
2
|
-
|
3
|
-
module Toiler
|
4
|
-
module Aws
|
5
|
-
# SQS Queue abstraction
|
6
|
-
# Provides methods for querying and acting on a SQS queue
|
7
|
-
class Queue
|
8
|
-
attr_accessor :name, :client, :url
|
9
|
-
|
10
|
-
def initialize(name, client = nil)
|
11
|
-
@name = name
|
12
|
-
@client = client || ::Aws::SQS::Client.new
|
13
|
-
@url = client.get_queue_url(queue_name: name).queue_url
|
14
|
-
end
|
15
|
-
|
16
|
-
def visibility_timeout
|
17
|
-
client.get_queue_attributes(
|
18
|
-
queue_url: url,
|
19
|
-
attribute_names: ['VisibilityTimeout']
|
20
|
-
).attributes['VisibilityTimeout'].to_i
|
21
|
-
end
|
22
|
-
|
23
|
-
def delete_messages(options)
|
24
|
-
client.delete_message_batch options.merge queue_url: url
|
25
|
-
end
|
26
|
-
|
27
|
-
def send_message(options)
|
28
|
-
client.send_message sanitize_message_body options.merge queue_url: url
|
29
|
-
end
|
30
|
-
|
31
|
-
def send_messages(options)
|
32
|
-
client.send_message_batch(
|
33
|
-
sanitize_message_body options.merge queue_url: url
|
34
|
-
)
|
35
|
-
end
|
36
|
-
|
37
|
-
def receive_messages(options)
|
38
|
-
client.receive_message(options.merge(queue_url: url))
|
39
|
-
.messages
|
40
|
-
.map { |m| Message.new(client, url, m) }
|
41
|
-
end
|
42
|
-
|
43
|
-
private
|
44
|
-
|
45
|
-
def sanitize_message_body(options)
|
46
|
-
messages = options[:entries] || [options]
|
47
|
-
|
48
|
-
messages.each do |m|
|
49
|
-
body = m[:message_body]
|
50
|
-
if body.is_a?(Hash)
|
51
|
-
m[:message_body] = JSON.dump(body)
|
52
|
-
elsif !body.is_a? String
|
53
|
-
fail ArgumentError, "Body must be a String, found #{body.class}"
|
54
|
-
end
|
55
|
-
end
|
56
|
-
|
57
|
-
options
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|
61
|
-
end
|
1
|
+
require 'toiler/aws/message'
|
2
|
+
|
3
|
+
module Toiler
|
4
|
+
module Aws
|
5
|
+
# SQS Queue abstraction
|
6
|
+
# Provides methods for querying and acting on a SQS queue
|
7
|
+
class Queue
|
8
|
+
attr_accessor :name, :client, :url
|
9
|
+
|
10
|
+
def initialize(name, client = nil)
|
11
|
+
@name = name
|
12
|
+
@client = client || ::Aws::SQS::Client.new
|
13
|
+
@url = client.get_queue_url(queue_name: name).queue_url
|
14
|
+
end
|
15
|
+
|
16
|
+
def visibility_timeout
|
17
|
+
client.get_queue_attributes(
|
18
|
+
queue_url: url,
|
19
|
+
attribute_names: ['VisibilityTimeout']
|
20
|
+
).attributes['VisibilityTimeout'].to_i
|
21
|
+
end
|
22
|
+
|
23
|
+
def delete_messages(options)
|
24
|
+
client.delete_message_batch options.merge queue_url: url
|
25
|
+
end
|
26
|
+
|
27
|
+
def send_message(options)
|
28
|
+
client.send_message sanitize_message_body options.merge queue_url: url
|
29
|
+
end
|
30
|
+
|
31
|
+
def send_messages(options)
|
32
|
+
client.send_message_batch(
|
33
|
+
sanitize_message_body options.merge queue_url: url
|
34
|
+
)
|
35
|
+
end
|
36
|
+
|
37
|
+
def receive_messages(options)
|
38
|
+
client.receive_message(options.merge(queue_url: url))
|
39
|
+
.messages
|
40
|
+
.map { |m| Message.new(client, url, m) }
|
41
|
+
end
|
42
|
+
|
43
|
+
private
|
44
|
+
|
45
|
+
def sanitize_message_body(options)
|
46
|
+
messages = options[:entries] || [options]
|
47
|
+
|
48
|
+
messages.each do |m|
|
49
|
+
body = m[:message_body]
|
50
|
+
if body.is_a?(Hash)
|
51
|
+
m[:message_body] = JSON.dump(body)
|
52
|
+
elsif !body.is_a? String
|
53
|
+
fail ArgumentError, "Body must be a String, found #{body.class}"
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
options
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
data/lib/toiler/cli.rb
CHANGED
@@ -1,164 +1,164 @@
|
|
1
|
-
require 'singleton'
|
2
|
-
require 'timeout'
|
3
|
-
require 'optparse'
|
4
|
-
require 'toiler'
|
5
|
-
|
6
|
-
module Toiler
|
7
|
-
# See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
|
8
|
-
class Shutdown < Interrupt; end
|
9
|
-
|
10
|
-
# Command line client interface
|
11
|
-
class CLI
|
12
|
-
include Singleton
|
13
|
-
|
14
|
-
attr_accessor :supervisor
|
15
|
-
|
16
|
-
def run(args)
|
17
|
-
@self_read, @self_write = IO.pipe
|
18
|
-
|
19
|
-
trap_signals
|
20
|
-
options = Utils::ArgumentParser.parse(args)
|
21
|
-
Utils::EnvironmentLoader.load(options)
|
22
|
-
daemonize
|
23
|
-
write_pid
|
24
|
-
load_concurrent
|
25
|
-
start_supervisor
|
26
|
-
|
27
|
-
handle_stop
|
28
|
-
end
|
29
|
-
|
30
|
-
private
|
31
|
-
|
32
|
-
def handle_stop
|
33
|
-
while (readable_io = IO.select([@self_read]))
|
34
|
-
handle_signal(readable_io.first[0].gets.strip)
|
35
|
-
end
|
36
|
-
rescue Interrupt
|
37
|
-
Toiler.logger.info 'Received Interrupt, Waiting up to 60 seconds for actors to finish...'
|
38
|
-
success = supervisor.ask(:terminate!).wait(60)
|
39
|
-
if success
|
40
|
-
Toiler.logger.info 'Supervisor successfully terminated'
|
41
|
-
else
|
42
|
-
Toiler.logger.info 'Timeout waiting dor Supervisor to terminate'
|
43
|
-
end
|
44
|
-
ensure
|
45
|
-
exit 0
|
46
|
-
end
|
47
|
-
|
48
|
-
def shutdown_pools
|
49
|
-
Concurrent.global_fast_executor.shutdown
|
50
|
-
Concurrent.global_io_executor.shutdown
|
51
|
-
return if Concurrent.global_io_executor.wait_for_termination(60)
|
52
|
-
Concurrent.global_io_executor.kill
|
53
|
-
end
|
54
|
-
|
55
|
-
def start_supervisor
|
56
|
-
require 'toiler/actor/supervisor'
|
57
|
-
@supervisor = Actor::Supervisor.spawn! :supervisor
|
58
|
-
end
|
59
|
-
|
60
|
-
def trap_signals
|
61
|
-
%w(INT TERM QUIT USR1 USR2 TTIN).each do |sig|
|
62
|
-
begin
|
63
|
-
trap sig do
|
64
|
-
@self_write.puts(sig)
|
65
|
-
end
|
66
|
-
rescue ArgumentError
|
67
|
-
puts "System does not support signal #{sig}"
|
68
|
-
end
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
72
|
-
def print_stacktraces
|
73
|
-
return unless Toiler.logger
|
74
|
-
Toiler.logger.info "-------------------"
|
75
|
-
Toiler.logger.info "Received QUIT, dumping threads:"
|
76
|
-
Thread.list.each do |t|
|
77
|
-
id = t.object_id
|
78
|
-
Toiler.logger.info "[thread:#{id}] #{t.backtrace.join("\n[thread:#{id}] ")}"
|
79
|
-
end
|
80
|
-
Toiler.logger.info '-------------------'
|
81
|
-
end
|
82
|
-
|
83
|
-
def print_status
|
84
|
-
return unless Toiler.logger
|
85
|
-
Toiler.logger.info "-------------------"
|
86
|
-
Toiler.logger.info "Received QUIT, dumping status:"
|
87
|
-
Toiler.queues.each do |queue|
|
88
|
-
fetcher = Toiler.fetcher(queue).send(:core).send(:context)
|
89
|
-
processor_pool = Toiler.processor_pool(queue).send(:core).send(:context)
|
90
|
-
processors = processor_pool.instance_variable_get(:@workers).collect{|w| w.send(:core).send(:context)}
|
91
|
-
busy_processors = processors.count{|pr| pr.executing?}
|
92
|
-
message = "Status for [queue:#{queue}]:"
|
93
|
-
message += "\n[fetcher:#{fetcher.name}] [executing:#{fetcher.executing?}] [polling:#{fetcher.polling?}] [scheduled:#{fetcher.scheduled?}] [free_processors:#{fetcher.get_free_processors}]"
|
94
|
-
message += "\n[processor_pool:#{processor_pool.name}] [workers:#{processors.count}] [busy:#{busy_processors}]"
|
95
|
-
processors.each do |processor|
|
96
|
-
thread = processor.thread
|
97
|
-
thread_id = thread.nil? ? "nil" : thread.object_id
|
98
|
-
message += "\n[processor:#{processor.name}] [executing:#{processor.executing?}] [thread:#{thread_id}]"
|
99
|
-
message += " Stack:\n" + thread.backtrace.join("\n\t") unless thread.nil?
|
100
|
-
end
|
101
|
-
Toiler.logger.info message
|
102
|
-
end
|
103
|
-
Toiler.logger.info '-------------------'
|
104
|
-
end
|
105
|
-
|
106
|
-
def handle_signal(signal)
|
107
|
-
case signal
|
108
|
-
when 'QUIT'
|
109
|
-
print_stacktraces
|
110
|
-
print_status
|
111
|
-
when 'INT', 'TERM'
|
112
|
-
fail Interrupt
|
113
|
-
end
|
114
|
-
end
|
115
|
-
|
116
|
-
def load_concurrent
|
117
|
-
require 'concurrent-edge'
|
118
|
-
Concurrent.global_logger = lambda do |level, progname, msg = nil, &block|
|
119
|
-
Toiler.logger.log(level, msg, progname, &block)
|
120
|
-
end if Toiler.logger
|
121
|
-
end
|
122
|
-
|
123
|
-
def daemonize
|
124
|
-
return unless Toiler.options[:daemon]
|
125
|
-
fail 'Logfile required when daemonizing' unless Toiler.options[:logfile]
|
126
|
-
|
127
|
-
files_to_reopen = []
|
128
|
-
ObjectSpace.each_object(File) do |file|
|
129
|
-
files_to_reopen << file unless file.closed?
|
130
|
-
end
|
131
|
-
|
132
|
-
Process.daemon(true, true)
|
133
|
-
|
134
|
-
reopen_files(files_to_reopen)
|
135
|
-
reopen_std
|
136
|
-
end
|
137
|
-
|
138
|
-
def reopen_files(files_to_reopen)
|
139
|
-
files_to_reopen.each do |file|
|
140
|
-
begin
|
141
|
-
file.reopen file.path, 'a+'
|
142
|
-
file.sync = true
|
143
|
-
rescue StandardError
|
144
|
-
puts "Failed to reopen file #{file}"
|
145
|
-
end
|
146
|
-
end
|
147
|
-
end
|
148
|
-
|
149
|
-
def reopen_std
|
150
|
-
[$stdout, $stderr].each do |io|
|
151
|
-
File.open(Toiler.options[:logfile], 'ab') do |f|
|
152
|
-
io.reopen(f)
|
153
|
-
end
|
154
|
-
io.sync = true
|
155
|
-
end
|
156
|
-
$stdin.reopen('/dev/null')
|
157
|
-
end
|
158
|
-
|
159
|
-
def write_pid
|
160
|
-
file = Toiler.options[:pidfile]
|
161
|
-
File.write file, Process.pid if file
|
162
|
-
end
|
163
|
-
end
|
164
|
-
end
|
1
|
+
require 'singleton'
|
2
|
+
require 'timeout'
|
3
|
+
require 'optparse'
|
4
|
+
require 'toiler'
|
5
|
+
|
6
|
+
module Toiler
|
7
|
+
# See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
|
8
|
+
class Shutdown < Interrupt; end
|
9
|
+
|
10
|
+
# Command line client interface
|
11
|
+
class CLI
|
12
|
+
include Singleton
|
13
|
+
|
14
|
+
attr_accessor :supervisor
|
15
|
+
|
16
|
+
def run(args)
|
17
|
+
@self_read, @self_write = IO.pipe
|
18
|
+
|
19
|
+
trap_signals
|
20
|
+
options = Utils::ArgumentParser.parse(args)
|
21
|
+
Utils::EnvironmentLoader.load(options)
|
22
|
+
daemonize
|
23
|
+
write_pid
|
24
|
+
load_concurrent
|
25
|
+
start_supervisor
|
26
|
+
|
27
|
+
handle_stop
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def handle_stop
|
33
|
+
while (readable_io = IO.select([@self_read]))
|
34
|
+
handle_signal(readable_io.first[0].gets.strip)
|
35
|
+
end
|
36
|
+
rescue Interrupt
|
37
|
+
Toiler.logger.info 'Received Interrupt, Waiting up to 60 seconds for actors to finish...'
|
38
|
+
success = supervisor.ask(:terminate!).wait(60)
|
39
|
+
if success
|
40
|
+
Toiler.logger.info 'Supervisor successfully terminated'
|
41
|
+
else
|
42
|
+
Toiler.logger.info 'Timeout waiting dor Supervisor to terminate'
|
43
|
+
end
|
44
|
+
ensure
|
45
|
+
exit 0
|
46
|
+
end
|
47
|
+
|
48
|
+
def shutdown_pools
|
49
|
+
Concurrent.global_fast_executor.shutdown
|
50
|
+
Concurrent.global_io_executor.shutdown
|
51
|
+
return if Concurrent.global_io_executor.wait_for_termination(60)
|
52
|
+
Concurrent.global_io_executor.kill
|
53
|
+
end
|
54
|
+
|
55
|
+
def start_supervisor
|
56
|
+
require 'toiler/actor/supervisor'
|
57
|
+
@supervisor = Actor::Supervisor.spawn! :supervisor
|
58
|
+
end
|
59
|
+
|
60
|
+
def trap_signals
|
61
|
+
%w(INT TERM QUIT USR1 USR2 TTIN).each do |sig|
|
62
|
+
begin
|
63
|
+
trap sig do
|
64
|
+
@self_write.puts(sig)
|
65
|
+
end
|
66
|
+
rescue ArgumentError
|
67
|
+
puts "System does not support signal #{sig}"
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def print_stacktraces
|
73
|
+
return unless Toiler.logger
|
74
|
+
Toiler.logger.info "-------------------"
|
75
|
+
Toiler.logger.info "Received QUIT, dumping threads:"
|
76
|
+
Thread.list.each do |t|
|
77
|
+
id = t.object_id
|
78
|
+
Toiler.logger.info "[thread:#{id}] #{t.backtrace.join("\n[thread:#{id}] ")}"
|
79
|
+
end
|
80
|
+
Toiler.logger.info '-------------------'
|
81
|
+
end
|
82
|
+
|
83
|
+
def print_status
|
84
|
+
return unless Toiler.logger
|
85
|
+
Toiler.logger.info "-------------------"
|
86
|
+
Toiler.logger.info "Received QUIT, dumping status:"
|
87
|
+
Toiler.queues.each do |queue|
|
88
|
+
fetcher = Toiler.fetcher(queue).send(:core).send(:context)
|
89
|
+
processor_pool = Toiler.processor_pool(queue).send(:core).send(:context)
|
90
|
+
processors = processor_pool.instance_variable_get(:@workers).collect{|w| w.send(:core).send(:context)}
|
91
|
+
busy_processors = processors.count{|pr| pr.executing?}
|
92
|
+
message = "Status for [queue:#{queue}]:"
|
93
|
+
message += "\n[fetcher:#{fetcher.name}] [executing:#{fetcher.executing?}] [polling:#{fetcher.polling?}] [scheduled:#{fetcher.scheduled?}] [free_processors:#{fetcher.get_free_processors}]"
|
94
|
+
message += "\n[processor_pool:#{processor_pool.name}] [workers:#{processors.count}] [busy:#{busy_processors}]"
|
95
|
+
processors.each do |processor|
|
96
|
+
thread = processor.thread
|
97
|
+
thread_id = thread.nil? ? "nil" : thread.object_id
|
98
|
+
message += "\n[processor:#{processor.name}] [executing:#{processor.executing?}] [thread:#{thread_id}]"
|
99
|
+
message += " Stack:\n" + thread.backtrace.join("\n\t") unless thread.nil?
|
100
|
+
end
|
101
|
+
Toiler.logger.info message
|
102
|
+
end
|
103
|
+
Toiler.logger.info '-------------------'
|
104
|
+
end
|
105
|
+
|
106
|
+
def handle_signal(signal)
|
107
|
+
case signal
|
108
|
+
when 'QUIT'
|
109
|
+
print_stacktraces
|
110
|
+
print_status
|
111
|
+
when 'INT', 'TERM'
|
112
|
+
fail Interrupt
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def load_concurrent
|
117
|
+
require 'concurrent-edge'
|
118
|
+
Concurrent.global_logger = lambda do |level, progname, msg = nil, &block|
|
119
|
+
Toiler.logger.log(level, msg, progname, &block)
|
120
|
+
end if Toiler.logger
|
121
|
+
end
|
122
|
+
|
123
|
+
def daemonize
|
124
|
+
return unless Toiler.options[:daemon]
|
125
|
+
fail 'Logfile required when daemonizing' unless Toiler.options[:logfile]
|
126
|
+
|
127
|
+
files_to_reopen = []
|
128
|
+
ObjectSpace.each_object(File) do |file|
|
129
|
+
files_to_reopen << file unless file.closed?
|
130
|
+
end
|
131
|
+
|
132
|
+
Process.daemon(true, true)
|
133
|
+
|
134
|
+
reopen_files(files_to_reopen)
|
135
|
+
reopen_std
|
136
|
+
end
|
137
|
+
|
138
|
+
def reopen_files(files_to_reopen)
|
139
|
+
files_to_reopen.each do |file|
|
140
|
+
begin
|
141
|
+
file.reopen file.path, 'a+'
|
142
|
+
file.sync = true
|
143
|
+
rescue StandardError
|
144
|
+
puts "Failed to reopen file #{file}"
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
def reopen_std
|
150
|
+
[$stdout, $stderr].each do |io|
|
151
|
+
File.open(Toiler.options[:logfile], 'ab') do |f|
|
152
|
+
io.reopen(f)
|
153
|
+
end
|
154
|
+
io.sync = true
|
155
|
+
end
|
156
|
+
$stdin.reopen('/dev/null')
|
157
|
+
end
|
158
|
+
|
159
|
+
def write_pid
|
160
|
+
file = Toiler.options[:pidfile]
|
161
|
+
File.write file, Process.pid if file
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|