toiler 0.3.6 → 0.4.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +35 -35
- data/.gitmodules +4 -4
- data/.ruby-version +1 -1
- data/Gemfile +9 -9
- data/Gemfile.lock +41 -41
- data/LICENSE +6 -6
- data/README.md +125 -125
- data/Rakefile +1 -1
- data/bin/toiler +12 -12
- data/lib/toiler.rb +55 -55
- data/lib/toiler/actor/fetcher.rb +116 -116
- data/lib/toiler/actor/processor.rb +124 -123
- data/lib/toiler/actor/supervisor.rb +55 -55
- data/lib/toiler/actor/utils/actor_logging.rb +28 -28
- data/lib/toiler/aws/message.rb +64 -64
- data/lib/toiler/aws/queue.rb +61 -61
- data/lib/toiler/cli.rb +164 -164
- data/lib/toiler/utils/argument_parser.rb +50 -50
- data/lib/toiler/utils/environment_loader.rb +104 -104
- data/lib/toiler/utils/logging.rb +41 -41
- data/lib/toiler/version.rb +4 -4
- data/lib/toiler/worker.rb +62 -62
- data/toiler.gemspec +30 -30
- metadata +4 -4
@@ -1,50 +1,50 @@
|
|
1
|
-
module Toiler
|
2
|
-
module Utils
|
3
|
-
# Parses command-line arguments
|
4
|
-
module ArgumentParser
|
5
|
-
module_function
|
6
|
-
|
7
|
-
def parse(argv)
|
8
|
-
opts = { queues: [] }
|
9
|
-
|
10
|
-
parser = OptionParser.new do |o|
|
11
|
-
o.on '-d', '--daemon', 'Daemonize process' do |arg|
|
12
|
-
opts[:daemon] = arg
|
13
|
-
end
|
14
|
-
|
15
|
-
o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
|
16
|
-
opts[:require] = arg
|
17
|
-
end
|
18
|
-
|
19
|
-
o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
|
20
|
-
opts[:config_file] = arg
|
21
|
-
end
|
22
|
-
|
23
|
-
o.on '-R', '--rails', 'Load Rails' do |arg|
|
24
|
-
opts[:rails] = arg
|
25
|
-
end
|
26
|
-
|
27
|
-
o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
|
28
|
-
opts[:logfile] = arg
|
29
|
-
end
|
30
|
-
|
31
|
-
o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
|
32
|
-
opts[:pidfile] = arg
|
33
|
-
end
|
34
|
-
|
35
|
-
o.on '-v', '--verbose', 'Print more verbose output' do |arg|
|
36
|
-
opts[:verbose] = arg
|
37
|
-
end
|
38
|
-
end
|
39
|
-
|
40
|
-
parser.banner = 'toiler [options]'
|
41
|
-
parser.on_tail '-h', '--help', 'Show help' do
|
42
|
-
puts parser
|
43
|
-
exit 1
|
44
|
-
end
|
45
|
-
parser.parse!(argv)
|
46
|
-
opts
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|
50
|
-
end
|
1
|
+
module Toiler
|
2
|
+
module Utils
|
3
|
+
# Parses command-line arguments
|
4
|
+
module ArgumentParser
|
5
|
+
module_function
|
6
|
+
|
7
|
+
def parse(argv)
|
8
|
+
opts = { queues: [] }
|
9
|
+
|
10
|
+
parser = OptionParser.new do |o|
|
11
|
+
o.on '-d', '--daemon', 'Daemonize process' do |arg|
|
12
|
+
opts[:daemon] = arg
|
13
|
+
end
|
14
|
+
|
15
|
+
o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
|
16
|
+
opts[:require] = arg
|
17
|
+
end
|
18
|
+
|
19
|
+
o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
|
20
|
+
opts[:config_file] = arg
|
21
|
+
end
|
22
|
+
|
23
|
+
o.on '-R', '--rails', 'Load Rails' do |arg|
|
24
|
+
opts[:rails] = arg
|
25
|
+
end
|
26
|
+
|
27
|
+
o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
|
28
|
+
opts[:logfile] = arg
|
29
|
+
end
|
30
|
+
|
31
|
+
o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
|
32
|
+
opts[:pidfile] = arg
|
33
|
+
end
|
34
|
+
|
35
|
+
o.on '-v', '--verbose', 'Print more verbose output' do |arg|
|
36
|
+
opts[:verbose] = arg
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
parser.banner = 'toiler [options]'
|
41
|
+
parser.on_tail '-h', '--help', 'Show help' do
|
42
|
+
puts parser
|
43
|
+
exit 1
|
44
|
+
end
|
45
|
+
parser.parse!(argv)
|
46
|
+
opts
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -1,104 +1,104 @@
|
|
1
|
-
require 'erb'
|
2
|
-
require 'yaml'
|
3
|
-
|
4
|
-
module Toiler
|
5
|
-
module Utils
|
6
|
-
# Takes care of loading componentes to get toiler ready to run
|
7
|
-
class EnvironmentLoader
|
8
|
-
attr_reader :options
|
9
|
-
|
10
|
-
def self.load(options)
|
11
|
-
new(options).load
|
12
|
-
end
|
13
|
-
|
14
|
-
def self.load_for_rails_console
|
15
|
-
load(config_file: (Rails.root + 'config' + 'toiler.yml'))
|
16
|
-
end
|
17
|
-
|
18
|
-
def initialize(options)
|
19
|
-
@options = options
|
20
|
-
end
|
21
|
-
|
22
|
-
def load
|
23
|
-
initialize_logger
|
24
|
-
load_rails if options[:rails]
|
25
|
-
require_workers if options[:require]
|
26
|
-
Toiler.options.merge!(config_file_options)
|
27
|
-
Toiler.options.merge!(options)
|
28
|
-
initialize_aws
|
29
|
-
end
|
30
|
-
|
31
|
-
private
|
32
|
-
|
33
|
-
def config_file_options
|
34
|
-
if (path = options[:config_file])
|
35
|
-
unless File.exist?(path)
|
36
|
-
Toiler.logger.warn "Config file #{path} does not exist"
|
37
|
-
path = nil
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
return {} unless path
|
42
|
-
|
43
|
-
deep_symbolize_keys YAML.load(ERB.new(File.read(path)).result)
|
44
|
-
end
|
45
|
-
|
46
|
-
def initialize_aws
|
47
|
-
return if Toiler.options[:aws].empty?
|
48
|
-
::Aws.config[:region] = Toiler.options[:aws][:region]
|
49
|
-
set_aws_credentials
|
50
|
-
end
|
51
|
-
|
52
|
-
def set_aws_credentials
|
53
|
-
return unless Toiler.options[:aws][:access_key_id]
|
54
|
-
::Aws.config[:credentials] = ::Aws::Credentials.new(
|
55
|
-
Toiler.options[:aws][:access_key_id],
|
56
|
-
Toiler.options[:aws][:secret_access_key]
|
57
|
-
)
|
58
|
-
end
|
59
|
-
|
60
|
-
def initialize_logger
|
61
|
-
Toiler::Utils::Logging.initialize_logger(options[:logfile])
|
62
|
-
Toiler.logger.level = Logger::DEBUG if options[:verbose]
|
63
|
-
end
|
64
|
-
|
65
|
-
def load_rails
|
66
|
-
require 'rails'
|
67
|
-
if ::Rails::VERSION::MAJOR < 4
|
68
|
-
load_rails_old
|
69
|
-
else
|
70
|
-
load_rails_new
|
71
|
-
end
|
72
|
-
Toiler.logger.info 'Rails environment loaded'
|
73
|
-
end
|
74
|
-
|
75
|
-
def load_rails_old
|
76
|
-
require File.expand_path('config/environment.rb')
|
77
|
-
::Rails.application.eager_load!
|
78
|
-
end
|
79
|
-
|
80
|
-
def load_rails_new
|
81
|
-
require File.expand_path('config/application.rb')
|
82
|
-
::Rails::Application.initializer 'toiler.eager_load' do
|
83
|
-
::Rails.application.config.eager_load = true
|
84
|
-
end
|
85
|
-
require File.expand_path('config/environment.rb')
|
86
|
-
end
|
87
|
-
|
88
|
-
def require_workers
|
89
|
-
require options[:require]
|
90
|
-
end
|
91
|
-
|
92
|
-
def deep_symbolize_keys(h)
|
93
|
-
h.each_with_object({}) do |(key, value), result|
|
94
|
-
k = key.respond_to?(:to_sym) ? key.to_sym : key
|
95
|
-
result[k] = if value.is_a? Hash
|
96
|
-
deep_symbolize_keys value
|
97
|
-
else
|
98
|
-
value
|
99
|
-
end
|
100
|
-
end
|
101
|
-
end
|
102
|
-
end
|
103
|
-
end
|
104
|
-
end
|
1
|
+
require 'erb'
|
2
|
+
require 'yaml'
|
3
|
+
|
4
|
+
module Toiler
|
5
|
+
module Utils
|
6
|
+
# Takes care of loading componentes to get toiler ready to run
|
7
|
+
class EnvironmentLoader
|
8
|
+
attr_reader :options
|
9
|
+
|
10
|
+
def self.load(options)
|
11
|
+
new(options).load
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.load_for_rails_console
|
15
|
+
load(config_file: (Rails.root + 'config' + 'toiler.yml'))
|
16
|
+
end
|
17
|
+
|
18
|
+
def initialize(options)
|
19
|
+
@options = options
|
20
|
+
end
|
21
|
+
|
22
|
+
def load
|
23
|
+
initialize_logger
|
24
|
+
load_rails if options[:rails]
|
25
|
+
require_workers if options[:require]
|
26
|
+
Toiler.options.merge!(config_file_options)
|
27
|
+
Toiler.options.merge!(options)
|
28
|
+
initialize_aws
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def config_file_options
|
34
|
+
if (path = options[:config_file])
|
35
|
+
unless File.exist?(path)
|
36
|
+
Toiler.logger.warn "Config file #{path} does not exist"
|
37
|
+
path = nil
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
return {} unless path
|
42
|
+
|
43
|
+
deep_symbolize_keys YAML.load(ERB.new(File.read(path)).result)
|
44
|
+
end
|
45
|
+
|
46
|
+
def initialize_aws
|
47
|
+
return if Toiler.options[:aws].empty?
|
48
|
+
::Aws.config[:region] = Toiler.options[:aws][:region]
|
49
|
+
set_aws_credentials
|
50
|
+
end
|
51
|
+
|
52
|
+
def set_aws_credentials
|
53
|
+
return unless Toiler.options[:aws][:access_key_id]
|
54
|
+
::Aws.config[:credentials] = ::Aws::Credentials.new(
|
55
|
+
Toiler.options[:aws][:access_key_id],
|
56
|
+
Toiler.options[:aws][:secret_access_key]
|
57
|
+
)
|
58
|
+
end
|
59
|
+
|
60
|
+
def initialize_logger
|
61
|
+
Toiler::Utils::Logging.initialize_logger(options[:logfile])
|
62
|
+
Toiler.logger.level = Logger::DEBUG if options[:verbose]
|
63
|
+
end
|
64
|
+
|
65
|
+
def load_rails
|
66
|
+
require 'rails'
|
67
|
+
if ::Rails::VERSION::MAJOR < 4
|
68
|
+
load_rails_old
|
69
|
+
else
|
70
|
+
load_rails_new
|
71
|
+
end
|
72
|
+
Toiler.logger.info 'Rails environment loaded'
|
73
|
+
end
|
74
|
+
|
75
|
+
def load_rails_old
|
76
|
+
require File.expand_path('config/environment.rb')
|
77
|
+
::Rails.application.eager_load!
|
78
|
+
end
|
79
|
+
|
80
|
+
def load_rails_new
|
81
|
+
require File.expand_path('config/application.rb')
|
82
|
+
::Rails::Application.initializer 'toiler.eager_load' do
|
83
|
+
::Rails.application.config.eager_load = true
|
84
|
+
end
|
85
|
+
require File.expand_path('config/environment.rb')
|
86
|
+
end
|
87
|
+
|
88
|
+
def require_workers
|
89
|
+
require options[:require]
|
90
|
+
end
|
91
|
+
|
92
|
+
def deep_symbolize_keys(h)
|
93
|
+
h.each_with_object({}) do |(key, value), result|
|
94
|
+
k = key.respond_to?(:to_sym) ? key.to_sym : key
|
95
|
+
result[k] = if value.is_a? Hash
|
96
|
+
deep_symbolize_keys value
|
97
|
+
else
|
98
|
+
value
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
data/lib/toiler/utils/logging.rb
CHANGED
@@ -1,41 +1,41 @@
|
|
1
|
-
require 'time'
|
2
|
-
require 'logger'
|
3
|
-
|
4
|
-
module Toiler
|
5
|
-
module Utils
|
6
|
-
# Initializes and exposes Toiler's default logger
|
7
|
-
module Logging
|
8
|
-
# Toiler's default log formatter
|
9
|
-
class Pretty < Logger::Formatter
|
10
|
-
def call(sev, time, progname, msg)
|
11
|
-
formatted = msg.respond_to?(:gsub) ? msg.gsub("\n", "\n\t") : msg
|
12
|
-
time = time.utc.iso8601
|
13
|
-
pid = Process.pid
|
14
|
-
if progname.to_s.empty?
|
15
|
-
"#{time} Pid:#{pid} Level:#{sev}: #{formatted}\n"
|
16
|
-
else
|
17
|
-
"#{time} Pid:#{pid} Actor:#{progname} Level:#{sev}: #{formatted}\n"
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
module_function
|
23
|
-
|
24
|
-
def initialize_logger(log_target = STDOUT)
|
25
|
-
log_target = STDOUT if log_target.nil?
|
26
|
-
@logger = Logger.new(log_target)
|
27
|
-
@logger.level = Logger::INFO
|
28
|
-
@logger.formatter = Pretty.new
|
29
|
-
@logger
|
30
|
-
end
|
31
|
-
|
32
|
-
def logger
|
33
|
-
@logger || initialize_logger
|
34
|
-
end
|
35
|
-
|
36
|
-
def logger=(log)
|
37
|
-
@logger = (log ? log : Logger.new('/dev/null'))
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
1
|
+
require 'time'
|
2
|
+
require 'logger'
|
3
|
+
|
4
|
+
module Toiler
|
5
|
+
module Utils
|
6
|
+
# Initializes and exposes Toiler's default logger
|
7
|
+
module Logging
|
8
|
+
# Toiler's default log formatter
|
9
|
+
class Pretty < Logger::Formatter
|
10
|
+
def call(sev, time, progname, msg)
|
11
|
+
formatted = msg.respond_to?(:gsub) ? msg.gsub("\n", "\n\t") : msg
|
12
|
+
time = time.utc.iso8601
|
13
|
+
pid = Process.pid
|
14
|
+
if progname.to_s.empty?
|
15
|
+
"#{time} Pid:#{pid} Level:#{sev}: #{formatted}\n"
|
16
|
+
else
|
17
|
+
"#{time} Pid:#{pid} Actor:#{progname} Level:#{sev}: #{formatted}\n"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
module_function
|
23
|
+
|
24
|
+
def initialize_logger(log_target = STDOUT)
|
25
|
+
log_target = STDOUT if log_target.nil?
|
26
|
+
@logger = Logger.new(log_target)
|
27
|
+
@logger.level = Logger::INFO
|
28
|
+
@logger.formatter = Pretty.new
|
29
|
+
@logger
|
30
|
+
end
|
31
|
+
|
32
|
+
def logger
|
33
|
+
@logger || initialize_logger
|
34
|
+
end
|
35
|
+
|
36
|
+
def logger=(log)
|
37
|
+
@logger = (log ? log : Logger.new('/dev/null'))
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
data/lib/toiler/version.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# Toiler Version
|
2
|
-
module Toiler
|
3
|
-
VERSION = '0.
|
4
|
-
end
|
1
|
+
# Toiler Version
|
2
|
+
module Toiler
|
3
|
+
VERSION = '0.4.0.beta1'.freeze
|
4
|
+
end
|
data/lib/toiler/worker.rb
CHANGED
@@ -1,62 +1,62 @@
|
|
1
|
-
module Toiler
|
2
|
-
# Toiler's Worker behaviour
|
3
|
-
module Worker
|
4
|
-
def self.included(base)
|
5
|
-
base.extend(ClassMethods)
|
6
|
-
base.class_variable_set(:@@toiler_options, Toiler.default_options)
|
7
|
-
end
|
8
|
-
|
9
|
-
def log(level, message)
|
10
|
-
Toiler.logger.log(level, message, self.class)
|
11
|
-
end
|
12
|
-
|
13
|
-
def error(msg)
|
14
|
-
log Logger::Severity::ERROR, msg
|
15
|
-
end
|
16
|
-
|
17
|
-
def info(msg)
|
18
|
-
log Logger::Severity::INFO, msg
|
19
|
-
end
|
20
|
-
|
21
|
-
def debug(msg)
|
22
|
-
log Logger::Severity::DEBUG, msg
|
23
|
-
end
|
24
|
-
|
25
|
-
def warn(msg)
|
26
|
-
log Logger::Severity::WARN, msg
|
27
|
-
end
|
28
|
-
|
29
|
-
def fatal(msg)
|
30
|
-
log Logger::Severity::FATAL, msg
|
31
|
-
end
|
32
|
-
|
33
|
-
# Class methods for Workers
|
34
|
-
module ClassMethods
|
35
|
-
def toiler_options(options = {})
|
36
|
-
return class_variable_get(:@@toiler_options) if options.empty?
|
37
|
-
Toiler.worker_class_registry[options[:queue]] = self if options[:queue]
|
38
|
-
class_variable_get(:@@toiler_options).merge! options
|
39
|
-
end
|
40
|
-
|
41
|
-
def batch?
|
42
|
-
class_variable_get(:@@toiler_options)[:batch]
|
43
|
-
end
|
44
|
-
|
45
|
-
def concurrency
|
46
|
-
class_variable_get(:@@toiler_options)[:concurrency]
|
47
|
-
end
|
48
|
-
|
49
|
-
def queue
|
50
|
-
class_variable_get(:@@toiler_options)[:queue]
|
51
|
-
end
|
52
|
-
|
53
|
-
def auto_visibility_timeout?
|
54
|
-
class_variable_get(:@@toiler_options)[:auto_visibility_timeout]
|
55
|
-
end
|
56
|
-
|
57
|
-
def auto_delete?
|
58
|
-
class_variable_get(:@@toiler_options)[:auto_delete]
|
59
|
-
end
|
60
|
-
end
|
61
|
-
end
|
62
|
-
end
|
1
|
+
module Toiler
|
2
|
+
# Toiler's Worker behaviour
|
3
|
+
module Worker
|
4
|
+
def self.included(base)
|
5
|
+
base.extend(ClassMethods)
|
6
|
+
base.class_variable_set(:@@toiler_options, Toiler.default_options)
|
7
|
+
end
|
8
|
+
|
9
|
+
def log(level, message)
|
10
|
+
Toiler.logger.log(level, message, self.class)
|
11
|
+
end
|
12
|
+
|
13
|
+
def error(msg)
|
14
|
+
log Logger::Severity::ERROR, msg
|
15
|
+
end
|
16
|
+
|
17
|
+
def info(msg)
|
18
|
+
log Logger::Severity::INFO, msg
|
19
|
+
end
|
20
|
+
|
21
|
+
def debug(msg)
|
22
|
+
log Logger::Severity::DEBUG, msg
|
23
|
+
end
|
24
|
+
|
25
|
+
def warn(msg)
|
26
|
+
log Logger::Severity::WARN, msg
|
27
|
+
end
|
28
|
+
|
29
|
+
def fatal(msg)
|
30
|
+
log Logger::Severity::FATAL, msg
|
31
|
+
end
|
32
|
+
|
33
|
+
# Class methods for Workers
|
34
|
+
module ClassMethods
|
35
|
+
def toiler_options(options = {})
|
36
|
+
return class_variable_get(:@@toiler_options) if options.empty?
|
37
|
+
Toiler.worker_class_registry[options[:queue]] = self if options[:queue]
|
38
|
+
class_variable_get(:@@toiler_options).merge! options
|
39
|
+
end
|
40
|
+
|
41
|
+
def batch?
|
42
|
+
class_variable_get(:@@toiler_options)[:batch]
|
43
|
+
end
|
44
|
+
|
45
|
+
def concurrency
|
46
|
+
class_variable_get(:@@toiler_options)[:concurrency]
|
47
|
+
end
|
48
|
+
|
49
|
+
def queue
|
50
|
+
class_variable_get(:@@toiler_options)[:queue]
|
51
|
+
end
|
52
|
+
|
53
|
+
def auto_visibility_timeout?
|
54
|
+
class_variable_get(:@@toiler_options)[:auto_visibility_timeout]
|
55
|
+
end
|
56
|
+
|
57
|
+
def auto_delete?
|
58
|
+
class_variable_get(:@@toiler_options)[:auto_delete]
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|