toiler 0.3.6 → 0.4.0.beta1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +35 -35
- data/.gitmodules +4 -4
- data/.ruby-version +1 -1
- data/Gemfile +9 -9
- data/Gemfile.lock +41 -41
- data/LICENSE +6 -6
- data/README.md +125 -125
- data/Rakefile +1 -1
- data/bin/toiler +12 -12
- data/lib/toiler.rb +55 -55
- data/lib/toiler/actor/fetcher.rb +116 -116
- data/lib/toiler/actor/processor.rb +124 -123
- data/lib/toiler/actor/supervisor.rb +55 -55
- data/lib/toiler/actor/utils/actor_logging.rb +28 -28
- data/lib/toiler/aws/message.rb +64 -64
- data/lib/toiler/aws/queue.rb +61 -61
- data/lib/toiler/cli.rb +164 -164
- data/lib/toiler/utils/argument_parser.rb +50 -50
- data/lib/toiler/utils/environment_loader.rb +104 -104
- data/lib/toiler/utils/logging.rb +41 -41
- data/lib/toiler/version.rb +4 -4
- data/lib/toiler/worker.rb +62 -62
- data/toiler.gemspec +30 -30
- metadata +4 -4
@@ -1,50 +1,50 @@
|
|
1
|
-
module Toiler
|
2
|
-
module Utils
|
3
|
-
# Parses command-line arguments
|
4
|
-
module ArgumentParser
|
5
|
-
module_function
|
6
|
-
|
7
|
-
def parse(argv)
|
8
|
-
opts = { queues: [] }
|
9
|
-
|
10
|
-
parser = OptionParser.new do |o|
|
11
|
-
o.on '-d', '--daemon', 'Daemonize process' do |arg|
|
12
|
-
opts[:daemon] = arg
|
13
|
-
end
|
14
|
-
|
15
|
-
o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
|
16
|
-
opts[:require] = arg
|
17
|
-
end
|
18
|
-
|
19
|
-
o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
|
20
|
-
opts[:config_file] = arg
|
21
|
-
end
|
22
|
-
|
23
|
-
o.on '-R', '--rails', 'Load Rails' do |arg|
|
24
|
-
opts[:rails] = arg
|
25
|
-
end
|
26
|
-
|
27
|
-
o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
|
28
|
-
opts[:logfile] = arg
|
29
|
-
end
|
30
|
-
|
31
|
-
o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
|
32
|
-
opts[:pidfile] = arg
|
33
|
-
end
|
34
|
-
|
35
|
-
o.on '-v', '--verbose', 'Print more verbose output' do |arg|
|
36
|
-
opts[:verbose] = arg
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
parser.banner = 'toiler [options]'
|
41
|
-
parser.on_tail '-h', '--help', 'Show help' do
|
42
|
-
puts parser
|
43
|
-
exit 1
|
44
|
-
end
|
45
|
-
parser.parse!(argv)
|
46
|
-
opts
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
end
|
1
|
+
module Toiler
|
2
|
+
module Utils
|
3
|
+
# Parses command-line arguments
|
4
|
+
module ArgumentParser
|
5
|
+
module_function
|
6
|
+
|
7
|
+
def parse(argv)
|
8
|
+
opts = { queues: [] }
|
9
|
+
|
10
|
+
parser = OptionParser.new do |o|
|
11
|
+
o.on '-d', '--daemon', 'Daemonize process' do |arg|
|
12
|
+
opts[:daemon] = arg
|
13
|
+
end
|
14
|
+
|
15
|
+
o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
|
16
|
+
opts[:require] = arg
|
17
|
+
end
|
18
|
+
|
19
|
+
o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
|
20
|
+
opts[:config_file] = arg
|
21
|
+
end
|
22
|
+
|
23
|
+
o.on '-R', '--rails', 'Load Rails' do |arg|
|
24
|
+
opts[:rails] = arg
|
25
|
+
end
|
26
|
+
|
27
|
+
o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
|
28
|
+
opts[:logfile] = arg
|
29
|
+
end
|
30
|
+
|
31
|
+
o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
|
32
|
+
opts[:pidfile] = arg
|
33
|
+
end
|
34
|
+
|
35
|
+
o.on '-v', '--verbose', 'Print more verbose output' do |arg|
|
36
|
+
opts[:verbose] = arg
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
parser.banner = 'toiler [options]'
|
41
|
+
parser.on_tail '-h', '--help', 'Show help' do
|
42
|
+
puts parser
|
43
|
+
exit 1
|
44
|
+
end
|
45
|
+
parser.parse!(argv)
|
46
|
+
opts
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -1,104 +1,104 @@
|
|
1
|
-
require 'erb'
|
2
|
-
require 'yaml'
|
3
|
-
|
4
|
-
module Toiler
|
5
|
-
module Utils
|
6
|
-
# Takes care of loading componentes to get toiler ready to run
|
7
|
-
class EnvironmentLoader
|
8
|
-
attr_reader :options
|
9
|
-
|
10
|
-
def self.load(options)
|
11
|
-
new(options).load
|
12
|
-
end
|
13
|
-
|
14
|
-
def self.load_for_rails_console
|
15
|
-
load(config_file: (Rails.root + 'config' + 'toiler.yml'))
|
16
|
-
end
|
17
|
-
|
18
|
-
def initialize(options)
|
19
|
-
@options = options
|
20
|
-
end
|
21
|
-
|
22
|
-
def load
|
23
|
-
initialize_logger
|
24
|
-
load_rails if options[:rails]
|
25
|
-
require_workers if options[:require]
|
26
|
-
Toiler.options.merge!(config_file_options)
|
27
|
-
Toiler.options.merge!(options)
|
28
|
-
initialize_aws
|
29
|
-
end
|
30
|
-
|
31
|
-
private
|
32
|
-
|
33
|
-
def config_file_options
|
34
|
-
if (path = options[:config_file])
|
35
|
-
unless File.exist?(path)
|
36
|
-
Toiler.logger.warn "Config file #{path} does not exist"
|
37
|
-
path = nil
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
return {} unless path
|
42
|
-
|
43
|
-
deep_symbolize_keys YAML.load(ERB.new(File.read(path)).result)
|
44
|
-
end
|
45
|
-
|
46
|
-
def initialize_aws
|
47
|
-
return if Toiler.options[:aws].empty?
|
48
|
-
::Aws.config[:region] = Toiler.options[:aws][:region]
|
49
|
-
set_aws_credentials
|
50
|
-
end
|
51
|
-
|
52
|
-
def set_aws_credentials
|
53
|
-
return unless Toiler.options[:aws][:access_key_id]
|
54
|
-
::Aws.config[:credentials] = ::Aws::Credentials.new(
|
55
|
-
Toiler.options[:aws][:access_key_id],
|
56
|
-
Toiler.options[:aws][:secret_access_key]
|
57
|
-
)
|
58
|
-
end
|
59
|
-
|
60
|
-
def initialize_logger
|
61
|
-
Toiler::Utils::Logging.initialize_logger(options[:logfile])
|
62
|
-
Toiler.logger.level = Logger::DEBUG if options[:verbose]
|
63
|
-
end
|
64
|
-
|
65
|
-
def load_rails
|
66
|
-
require 'rails'
|
67
|
-
if ::Rails::VERSION::MAJOR < 4
|
68
|
-
load_rails_old
|
69
|
-
else
|
70
|
-
load_rails_new
|
71
|
-
end
|
72
|
-
Toiler.logger.info 'Rails environment loaded'
|
73
|
-
end
|
74
|
-
|
75
|
-
def load_rails_old
|
76
|
-
require File.expand_path('config/environment.rb')
|
77
|
-
::Rails.application.eager_load!
|
78
|
-
end
|
79
|
-
|
80
|
-
def load_rails_new
|
81
|
-
require File.expand_path('config/application.rb')
|
82
|
-
::Rails::Application.initializer 'toiler.eager_load' do
|
83
|
-
::Rails.application.config.eager_load = true
|
84
|
-
end
|
85
|
-
require File.expand_path('config/environment.rb')
|
86
|
-
end
|
87
|
-
|
88
|
-
def require_workers
|
89
|
-
require options[:require]
|
90
|
-
end
|
91
|
-
|
92
|
-
def deep_symbolize_keys(h)
|
93
|
-
h.each_with_object({}) do |(key, value), result|
|
94
|
-
k = key.respond_to?(:to_sym) ? key.to_sym : key
|
95
|
-
result[k] = if value.is_a? Hash
|
96
|
-
deep_symbolize_keys value
|
97
|
-
else
|
98
|
-
value
|
99
|
-
end
|
100
|
-
end
|
101
|
-
end
|
102
|
-
end
|
103
|
-
end
|
104
|
-
end
|
1
|
+
require 'erb'
|
2
|
+
require 'yaml'
|
3
|
+
|
4
|
+
module Toiler
|
5
|
+
module Utils
|
6
|
+
# Takes care of loading componentes to get toiler ready to run
|
7
|
+
class EnvironmentLoader
|
8
|
+
attr_reader :options
|
9
|
+
|
10
|
+
def self.load(options)
|
11
|
+
new(options).load
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.load_for_rails_console
|
15
|
+
load(config_file: (Rails.root + 'config' + 'toiler.yml'))
|
16
|
+
end
|
17
|
+
|
18
|
+
def initialize(options)
|
19
|
+
@options = options
|
20
|
+
end
|
21
|
+
|
22
|
+
def load
|
23
|
+
initialize_logger
|
24
|
+
load_rails if options[:rails]
|
25
|
+
require_workers if options[:require]
|
26
|
+
Toiler.options.merge!(config_file_options)
|
27
|
+
Toiler.options.merge!(options)
|
28
|
+
initialize_aws
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def config_file_options
|
34
|
+
if (path = options[:config_file])
|
35
|
+
unless File.exist?(path)
|
36
|
+
Toiler.logger.warn "Config file #{path} does not exist"
|
37
|
+
path = nil
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
return {} unless path
|
42
|
+
|
43
|
+
deep_symbolize_keys YAML.load(ERB.new(File.read(path)).result)
|
44
|
+
end
|
45
|
+
|
46
|
+
def initialize_aws
|
47
|
+
return if Toiler.options[:aws].empty?
|
48
|
+
::Aws.config[:region] = Toiler.options[:aws][:region]
|
49
|
+
set_aws_credentials
|
50
|
+
end
|
51
|
+
|
52
|
+
def set_aws_credentials
|
53
|
+
return unless Toiler.options[:aws][:access_key_id]
|
54
|
+
::Aws.config[:credentials] = ::Aws::Credentials.new(
|
55
|
+
Toiler.options[:aws][:access_key_id],
|
56
|
+
Toiler.options[:aws][:secret_access_key]
|
57
|
+
)
|
58
|
+
end
|
59
|
+
|
60
|
+
def initialize_logger
|
61
|
+
Toiler::Utils::Logging.initialize_logger(options[:logfile])
|
62
|
+
Toiler.logger.level = Logger::DEBUG if options[:verbose]
|
63
|
+
end
|
64
|
+
|
65
|
+
def load_rails
|
66
|
+
require 'rails'
|
67
|
+
if ::Rails::VERSION::MAJOR < 4
|
68
|
+
load_rails_old
|
69
|
+
else
|
70
|
+
load_rails_new
|
71
|
+
end
|
72
|
+
Toiler.logger.info 'Rails environment loaded'
|
73
|
+
end
|
74
|
+
|
75
|
+
def load_rails_old
|
76
|
+
require File.expand_path('config/environment.rb')
|
77
|
+
::Rails.application.eager_load!
|
78
|
+
end
|
79
|
+
|
80
|
+
def load_rails_new
|
81
|
+
require File.expand_path('config/application.rb')
|
82
|
+
::Rails::Application.initializer 'toiler.eager_load' do
|
83
|
+
::Rails.application.config.eager_load = true
|
84
|
+
end
|
85
|
+
require File.expand_path('config/environment.rb')
|
86
|
+
end
|
87
|
+
|
88
|
+
def require_workers
|
89
|
+
require options[:require]
|
90
|
+
end
|
91
|
+
|
92
|
+
def deep_symbolize_keys(h)
|
93
|
+
h.each_with_object({}) do |(key, value), result|
|
94
|
+
k = key.respond_to?(:to_sym) ? key.to_sym : key
|
95
|
+
result[k] = if value.is_a? Hash
|
96
|
+
deep_symbolize_keys value
|
97
|
+
else
|
98
|
+
value
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
data/lib/toiler/utils/logging.rb
CHANGED
@@ -1,41 +1,41 @@
|
|
1
|
-
require 'time'
|
2
|
-
require 'logger'
|
3
|
-
|
4
|
-
module Toiler
|
5
|
-
module Utils
|
6
|
-
# Initializes and exposes Toiler's default logger
|
7
|
-
module Logging
|
8
|
-
# Toiler's default log formatter
|
9
|
-
class Pretty < Logger::Formatter
|
10
|
-
def call(sev, time, progname, msg)
|
11
|
-
formatted = msg.respond_to?(:gsub) ? msg.gsub("\n", "\n\t") : msg
|
12
|
-
time = time.utc.iso8601
|
13
|
-
pid = Process.pid
|
14
|
-
if progname.to_s.empty?
|
15
|
-
"#{time} Pid:#{pid} Level:#{sev}: #{formatted}\n"
|
16
|
-
else
|
17
|
-
"#{time} Pid:#{pid} Actor:#{progname} Level:#{sev}: #{formatted}\n"
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
module_function
|
23
|
-
|
24
|
-
def initialize_logger(log_target = STDOUT)
|
25
|
-
log_target = STDOUT if log_target.nil?
|
26
|
-
@logger = Logger.new(log_target)
|
27
|
-
@logger.level = Logger::INFO
|
28
|
-
@logger.formatter = Pretty.new
|
29
|
-
@logger
|
30
|
-
end
|
31
|
-
|
32
|
-
def logger
|
33
|
-
@logger || initialize_logger
|
34
|
-
end
|
35
|
-
|
36
|
-
def logger=(log)
|
37
|
-
@logger = (log ? log : Logger.new('/dev/null'))
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
1
|
+
require 'time'
|
2
|
+
require 'logger'
|
3
|
+
|
4
|
+
module Toiler
|
5
|
+
module Utils
|
6
|
+
# Initializes and exposes Toiler's default logger
|
7
|
+
module Logging
|
8
|
+
# Toiler's default log formatter
|
9
|
+
class Pretty < Logger::Formatter
|
10
|
+
def call(sev, time, progname, msg)
|
11
|
+
formatted = msg.respond_to?(:gsub) ? msg.gsub("\n", "\n\t") : msg
|
12
|
+
time = time.utc.iso8601
|
13
|
+
pid = Process.pid
|
14
|
+
if progname.to_s.empty?
|
15
|
+
"#{time} Pid:#{pid} Level:#{sev}: #{formatted}\n"
|
16
|
+
else
|
17
|
+
"#{time} Pid:#{pid} Actor:#{progname} Level:#{sev}: #{formatted}\n"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
module_function
|
23
|
+
|
24
|
+
def initialize_logger(log_target = STDOUT)
|
25
|
+
log_target = STDOUT if log_target.nil?
|
26
|
+
@logger = Logger.new(log_target)
|
27
|
+
@logger.level = Logger::INFO
|
28
|
+
@logger.formatter = Pretty.new
|
29
|
+
@logger
|
30
|
+
end
|
31
|
+
|
32
|
+
def logger
|
33
|
+
@logger || initialize_logger
|
34
|
+
end
|
35
|
+
|
36
|
+
def logger=(log)
|
37
|
+
@logger = (log ? log : Logger.new('/dev/null'))
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
data/lib/toiler/version.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Toiler Version
|
2
|
-
module Toiler
|
3
|
-
VERSION = '0.
|
4
|
-
end
|
1
|
+
# Toiler Version
|
2
|
+
module Toiler
|
3
|
+
VERSION = '0.4.0.beta1'.freeze
|
4
|
+
end
|
data/lib/toiler/worker.rb
CHANGED
@@ -1,62 +1,62 @@
|
|
1
|
-
module Toiler
|
2
|
-
# Toiler's Worker behaviour
|
3
|
-
module Worker
|
4
|
-
def self.included(base)
|
5
|
-
base.extend(ClassMethods)
|
6
|
-
base.class_variable_set(:@@toiler_options, Toiler.default_options)
|
7
|
-
end
|
8
|
-
|
9
|
-
def log(level, message)
|
10
|
-
Toiler.logger.log(level, message, self.class)
|
11
|
-
end
|
12
|
-
|
13
|
-
def error(msg)
|
14
|
-
log Logger::Severity::ERROR, msg
|
15
|
-
end
|
16
|
-
|
17
|
-
def info(msg)
|
18
|
-
log Logger::Severity::INFO, msg
|
19
|
-
end
|
20
|
-
|
21
|
-
def debug(msg)
|
22
|
-
log Logger::Severity::DEBUG, msg
|
23
|
-
end
|
24
|
-
|
25
|
-
def warn(msg)
|
26
|
-
log Logger::Severity::WARN, msg
|
27
|
-
end
|
28
|
-
|
29
|
-
def fatal(msg)
|
30
|
-
log Logger::Severity::FATAL, msg
|
31
|
-
end
|
32
|
-
|
33
|
-
# Class methods for Workers
|
34
|
-
module ClassMethods
|
35
|
-
def toiler_options(options = {})
|
36
|
-
return class_variable_get(:@@toiler_options) if options.empty?
|
37
|
-
Toiler.worker_class_registry[options[:queue]] = self if options[:queue]
|
38
|
-
class_variable_get(:@@toiler_options).merge! options
|
39
|
-
end
|
40
|
-
|
41
|
-
def batch?
|
42
|
-
class_variable_get(:@@toiler_options)[:batch]
|
43
|
-
end
|
44
|
-
|
45
|
-
def concurrency
|
46
|
-
class_variable_get(:@@toiler_options)[:concurrency]
|
47
|
-
end
|
48
|
-
|
49
|
-
def queue
|
50
|
-
class_variable_get(:@@toiler_options)[:queue]
|
51
|
-
end
|
52
|
-
|
53
|
-
def auto_visibility_timeout?
|
54
|
-
class_variable_get(:@@toiler_options)[:auto_visibility_timeout]
|
55
|
-
end
|
56
|
-
|
57
|
-
def auto_delete?
|
58
|
-
class_variable_get(:@@toiler_options)[:auto_delete]
|
59
|
-
end
|
60
|
-
end
|
61
|
-
end
|
62
|
-
end
|
1
|
+
module Toiler
|
2
|
+
# Toiler's Worker behaviour
|
3
|
+
module Worker
|
4
|
+
def self.included(base)
|
5
|
+
base.extend(ClassMethods)
|
6
|
+
base.class_variable_set(:@@toiler_options, Toiler.default_options)
|
7
|
+
end
|
8
|
+
|
9
|
+
def log(level, message)
|
10
|
+
Toiler.logger.log(level, message, self.class)
|
11
|
+
end
|
12
|
+
|
13
|
+
def error(msg)
|
14
|
+
log Logger::Severity::ERROR, msg
|
15
|
+
end
|
16
|
+
|
17
|
+
def info(msg)
|
18
|
+
log Logger::Severity::INFO, msg
|
19
|
+
end
|
20
|
+
|
21
|
+
def debug(msg)
|
22
|
+
log Logger::Severity::DEBUG, msg
|
23
|
+
end
|
24
|
+
|
25
|
+
def warn(msg)
|
26
|
+
log Logger::Severity::WARN, msg
|
27
|
+
end
|
28
|
+
|
29
|
+
def fatal(msg)
|
30
|
+
log Logger::Severity::FATAL, msg
|
31
|
+
end
|
32
|
+
|
33
|
+
# Class methods for Workers
|
34
|
+
module ClassMethods
|
35
|
+
def toiler_options(options = {})
|
36
|
+
return class_variable_get(:@@toiler_options) if options.empty?
|
37
|
+
Toiler.worker_class_registry[options[:queue]] = self if options[:queue]
|
38
|
+
class_variable_get(:@@toiler_options).merge! options
|
39
|
+
end
|
40
|
+
|
41
|
+
def batch?
|
42
|
+
class_variable_get(:@@toiler_options)[:batch]
|
43
|
+
end
|
44
|
+
|
45
|
+
def concurrency
|
46
|
+
class_variable_get(:@@toiler_options)[:concurrency]
|
47
|
+
end
|
48
|
+
|
49
|
+
def queue
|
50
|
+
class_variable_get(:@@toiler_options)[:queue]
|
51
|
+
end
|
52
|
+
|
53
|
+
def auto_visibility_timeout?
|
54
|
+
class_variable_get(:@@toiler_options)[:auto_visibility_timeout]
|
55
|
+
end
|
56
|
+
|
57
|
+
def auto_delete?
|
58
|
+
class_variable_get(:@@toiler_options)[:auto_delete]
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|