toiler 0.3.1.beta1 → 0.3.1.beta2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,104 +1,104 @@
1
- require 'erb'
2
- require 'yaml'
3
-
4
- module Toiler
5
- module Utils
6
- # Takes care of loading componentes to get toiler ready to run
7
- class EnvironmentLoader
8
- attr_reader :options
9
-
10
- def self.load(options)
11
- new(options).load
12
- end
13
-
14
- def self.load_for_rails_console
15
- load(config_file: (Rails.root + 'config' + 'toiler.yml'))
16
- end
17
-
18
- def initialize(options)
19
- @options = options
20
- end
21
-
22
- def load
23
- initialize_logger
24
- load_rails if options[:rails]
25
- require_workers if options[:require]
26
- Toiler.options.merge!(config_file_options)
27
- Toiler.options.merge!(options)
28
- initialize_aws
29
- end
30
-
31
- private
32
-
33
- def config_file_options
34
- if (path = options[:config_file])
35
- unless File.exist?(path)
36
- Toiler.logger.warn "Config file #{path} does not exist"
37
- path = nil
38
- end
39
- end
40
-
41
- return {} unless path
42
-
43
- deep_symbolize_keys YAML.load(ERB.new(File.read(path)).result)
44
- end
45
-
46
- def initialize_aws
47
- return if Toiler.options[:aws].empty?
48
- ::Aws.config[:region] = Toiler.options[:aws][:region]
49
- set_aws_credentials
50
- end
51
-
52
- def set_aws_credentials
53
- return unless Toiler.options[:aws][:access_key_id]
54
- ::Aws.config[:credentials] = ::Aws::Credentials.new(
55
- Toiler.options[:aws][:access_key_id],
56
- Toiler.options[:aws][:secret_access_key]
57
- )
58
- end
59
-
60
- def initialize_logger
61
- Toiler::Utils::Logging.initialize_logger(options[:logfile])
62
- Toiler.logger.level = Logger::DEBUG if options[:verbose]
63
- end
64
-
65
- def load_rails
66
- require 'rails'
67
- if ::Rails::VERSION::MAJOR < 4
68
- load_rails_old
69
- else
70
- load_rails_new
71
- end
72
- Toiler.logger.info 'Rails environment loaded'
73
- end
74
-
75
- def load_rails_old
76
- require File.expand_path('config/environment.rb')
77
- ::Rails.application.eager_load!
78
- end
79
-
80
- def load_rails_new
81
- require File.expand_path('config/application.rb')
82
- ::Rails::Application.initializer 'toiler.eager_load' do
83
- ::Rails.application.config.eager_load = true
84
- end
85
- require File.expand_path('config/environment.rb')
86
- end
87
-
88
- def require_workers
89
- require options[:require]
90
- end
91
-
92
- def deep_symbolize_keys(h)
93
- h.each_with_object({}) do |(key, value), result|
94
- k = key.respond_to?(:to_sym) ? key.to_sym : key
95
- result[k] = if value.is_a? Hash
96
- deep_symbolize_keys value
97
- else
98
- value
99
- end
100
- end
101
- end
102
- end
103
- end
104
- end
1
+ require 'erb'
2
+ require 'yaml'
3
+
4
+ module Toiler
5
+ module Utils
6
+ # Takes care of loading componentes to get toiler ready to run
7
+ class EnvironmentLoader
8
+ attr_reader :options
9
+
10
+ def self.load(options)
11
+ new(options).load
12
+ end
13
+
14
+ def self.load_for_rails_console
15
+ load(config_file: (Rails.root + 'config' + 'toiler.yml'))
16
+ end
17
+
18
+ def initialize(options)
19
+ @options = options
20
+ end
21
+
22
+ def load
23
+ initialize_logger
24
+ load_rails if options[:rails]
25
+ require_workers if options[:require]
26
+ Toiler.options.merge!(config_file_options)
27
+ Toiler.options.merge!(options)
28
+ initialize_aws
29
+ end
30
+
31
+ private
32
+
33
+ def config_file_options
34
+ if (path = options[:config_file])
35
+ unless File.exist?(path)
36
+ Toiler.logger.warn "Config file #{path} does not exist"
37
+ path = nil
38
+ end
39
+ end
40
+
41
+ return {} unless path
42
+
43
+ deep_symbolize_keys YAML.load(ERB.new(File.read(path)).result)
44
+ end
45
+
46
+ def initialize_aws
47
+ return if Toiler.options[:aws].empty?
48
+ ::Aws.config[:region] = Toiler.options[:aws][:region]
49
+ set_aws_credentials
50
+ end
51
+
52
+ def set_aws_credentials
53
+ return unless Toiler.options[:aws][:access_key_id]
54
+ ::Aws.config[:credentials] = ::Aws::Credentials.new(
55
+ Toiler.options[:aws][:access_key_id],
56
+ Toiler.options[:aws][:secret_access_key]
57
+ )
58
+ end
59
+
60
+ def initialize_logger
61
+ Toiler::Utils::Logging.initialize_logger(options[:logfile])
62
+ Toiler.logger.level = Logger::DEBUG if options[:verbose]
63
+ end
64
+
65
+ def load_rails
66
+ require 'rails'
67
+ if ::Rails::VERSION::MAJOR < 4
68
+ load_rails_old
69
+ else
70
+ load_rails_new
71
+ end
72
+ Toiler.logger.info 'Rails environment loaded'
73
+ end
74
+
75
+ def load_rails_old
76
+ require File.expand_path('config/environment.rb')
77
+ ::Rails.application.eager_load!
78
+ end
79
+
80
+ def load_rails_new
81
+ require File.expand_path('config/application.rb')
82
+ ::Rails::Application.initializer 'toiler.eager_load' do
83
+ ::Rails.application.config.eager_load = true
84
+ end
85
+ require File.expand_path('config/environment.rb')
86
+ end
87
+
88
+ def require_workers
89
+ require options[:require]
90
+ end
91
+
92
+ def deep_symbolize_keys(h)
93
+ h.each_with_object({}) do |(key, value), result|
94
+ k = key.respond_to?(:to_sym) ? key.to_sym : key
95
+ result[k] = if value.is_a? Hash
96
+ deep_symbolize_keys value
97
+ else
98
+ value
99
+ end
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
@@ -1,41 +1,41 @@
1
- require 'time'
2
- require 'logger'
3
-
4
- module Toiler
5
- module Utils
6
- # Initializes and exposes Toiler's default logger
7
- module Logging
8
- # Toiler's default log formatter
9
- class Pretty < Logger::Formatter
10
- def call(sev, time, progname, msg)
11
- formatted = msg.respond_to?(:gsub) ? msg.gsub("\n", "\n\t") : msg
12
- time = time.utc.iso8601
13
- pid = Process.pid
14
- if progname.to_s.empty?
15
- "#{time} Pid:#{pid} Level:#{sev}: #{formatted}\n"
16
- else
17
- "#{time} Pid:#{pid} Actor:#{progname} Level:#{sev}: #{formatted}\n"
18
- end
19
- end
20
- end
21
-
22
- module_function
23
-
24
- def initialize_logger(log_target = STDOUT)
25
- log_target = STDOUT if log_target.nil?
26
- @logger = Logger.new(log_target)
27
- @logger.level = Logger::INFO
28
- @logger.formatter = Pretty.new
29
- @logger
30
- end
31
-
32
- def logger
33
- @logger || initialize_logger
34
- end
35
-
36
- def logger=(log)
37
- @logger = (log ? log : Logger.new('/dev/null'))
38
- end
39
- end
40
- end
41
- end
1
+ require 'time'
2
+ require 'logger'
3
+
4
+ module Toiler
5
+ module Utils
6
+ # Initializes and exposes Toiler's default logger
7
+ module Logging
8
+ # Toiler's default log formatter
9
+ class Pretty < Logger::Formatter
10
+ def call(sev, time, progname, msg)
11
+ formatted = msg.respond_to?(:gsub) ? msg.gsub("\n", "\n\t") : msg
12
+ time = time.utc.iso8601
13
+ pid = Process.pid
14
+ if progname.to_s.empty?
15
+ "#{time} Pid:#{pid} Level:#{sev}: #{formatted}\n"
16
+ else
17
+ "#{time} Pid:#{pid} Actor:#{progname} Level:#{sev}: #{formatted}\n"
18
+ end
19
+ end
20
+ end
21
+
22
+ module_function
23
+
24
+ def initialize_logger(log_target = STDOUT)
25
+ log_target = STDOUT if log_target.nil?
26
+ @logger = Logger.new(log_target)
27
+ @logger.level = Logger::INFO
28
+ @logger.formatter = Pretty.new
29
+ @logger
30
+ end
31
+
32
+ def logger
33
+ @logger || initialize_logger
34
+ end
35
+
36
+ def logger=(log)
37
+ @logger = (log ? log : Logger.new('/dev/null'))
38
+ end
39
+ end
40
+ end
41
+ end
@@ -1,4 +1,4 @@
1
- # Toiler Version
2
- module Toiler
3
- VERSION = '0.3.1.beta1'.freeze
4
- end
1
+ # Toiler Version
2
+ module Toiler
3
+ VERSION = '0.3.1.beta2'.freeze
4
+ end
data/lib/toiler/worker.rb CHANGED
@@ -1,62 +1,62 @@
1
- module Toiler
2
- # Toiler's Worker behaviour
3
- module Worker
4
- def self.included(base)
5
- base.extend(ClassMethods)
6
- base.class_variable_set(:@@toiler_options, Toiler.default_options)
7
- end
8
-
9
- def log(level, message)
10
- Toiler.logger.log(level, message, self.class)
11
- end
12
-
13
- def error(msg)
14
- log Logger::Severity::ERROR, msg
15
- end
16
-
17
- def info(msg)
18
- log Logger::Severity::INFO, msg
19
- end
20
-
21
- def debug(msg)
22
- log Logger::Severity::DEBUG, msg
23
- end
24
-
25
- def warn(msg)
26
- log Logger::Severity::WARN, msg
27
- end
28
-
29
- def fatal(msg)
30
- log Logger::Severity::FATAL, msg
31
- end
32
-
33
- # Class methods for Workers
34
- module ClassMethods
35
- def toiler_options(options = {})
36
- return class_variable_get(:@@toiler_options) if options.empty?
37
- Toiler.worker_class_registry[options[:queue]] = self if options[:queue]
38
- class_variable_get(:@@toiler_options).merge! options
39
- end
40
-
41
- def batch?
42
- class_variable_get(:@@toiler_options)[:batch]
43
- end
44
-
45
- def concurrency
46
- class_variable_get(:@@toiler_options)[:concurrency]
47
- end
48
-
49
- def queue
50
- class_variable_get(:@@toiler_options)[:queue]
51
- end
52
-
53
- def auto_visibility_timeout?
54
- class_variable_get(:@@toiler_options)[:auto_visibility_timeout]
55
- end
56
-
57
- def auto_delete?
58
- class_variable_get(:@@toiler_options)[:auto_delete]
59
- end
60
- end
61
- end
62
- end
1
+ module Toiler
2
+ # Toiler's Worker behaviour
3
+ module Worker
4
+ def self.included(base)
5
+ base.extend(ClassMethods)
6
+ base.class_variable_set(:@@toiler_options, Toiler.default_options)
7
+ end
8
+
9
+ def log(level, message)
10
+ Toiler.logger.log(level, message, self.class)
11
+ end
12
+
13
+ def error(msg)
14
+ log Logger::Severity::ERROR, msg
15
+ end
16
+
17
+ def info(msg)
18
+ log Logger::Severity::INFO, msg
19
+ end
20
+
21
+ def debug(msg)
22
+ log Logger::Severity::DEBUG, msg
23
+ end
24
+
25
+ def warn(msg)
26
+ log Logger::Severity::WARN, msg
27
+ end
28
+
29
+ def fatal(msg)
30
+ log Logger::Severity::FATAL, msg
31
+ end
32
+
33
+ # Class methods for Workers
34
+ module ClassMethods
35
+ def toiler_options(options = {})
36
+ return class_variable_get(:@@toiler_options) if options.empty?
37
+ Toiler.worker_class_registry[options[:queue]] = self if options[:queue]
38
+ class_variable_get(:@@toiler_options).merge! options
39
+ end
40
+
41
+ def batch?
42
+ class_variable_get(:@@toiler_options)[:batch]
43
+ end
44
+
45
+ def concurrency
46
+ class_variable_get(:@@toiler_options)[:concurrency]
47
+ end
48
+
49
+ def queue
50
+ class_variable_get(:@@toiler_options)[:queue]
51
+ end
52
+
53
+ def auto_visibility_timeout?
54
+ class_variable_get(:@@toiler_options)[:auto_visibility_timeout]
55
+ end
56
+
57
+ def auto_delete?
58
+ class_variable_get(:@@toiler_options)[:auto_delete]
59
+ end
60
+ end
61
+ end
62
+ end
data/lib/toiler.rb CHANGED
@@ -1,55 +1,55 @@
1
- require 'aws-sdk'
2
- require 'toiler/utils/environment_loader'
3
- require 'toiler/utils/logging'
4
- require 'toiler/utils/argument_parser'
5
- require 'toiler/worker'
6
- require 'toiler/cli'
7
- require 'toiler/version'
8
-
9
- # Main module
10
- module Toiler
11
- @worker_class_registry = {}
12
- @options = {
13
- aws: {}
14
- }
15
- @fetchers = {}
16
- @processor_pools = {}
17
-
18
- attr_reader :worker_class_registry, :options, :fetchers, :processor_pools
19
- module_function :worker_class_registry, :options, :fetchers, :processor_pools
20
-
21
- module_function
22
-
23
- def logger
24
- Toiler::Utils::Logging.logger
25
- end
26
-
27
- def queues
28
- worker_class_registry.keys
29
- end
30
-
31
- def fetcher(queue)
32
- fetchers["fetcher_#{queue}".to_sym]
33
- end
34
-
35
- def set_fetcher(queue, val)
36
- fetchers["fetcher_#{queue}".to_sym] = val
37
- end
38
-
39
- def processor_pool(queue)
40
- processor_pools["processor_pool_#{queue}".to_sym]
41
- end
42
-
43
- def set_processor_pool(queue, val)
44
- processor_pools["processor_pool_#{queue}".to_sym] = val
45
- end
46
-
47
- def default_options
48
- {
49
- auto_visibility_timeout: false,
50
- concurrency: 1,
51
- auto_delete: false,
52
- batch: false
53
- }
54
- end
55
- end
1
+ require 'aws-sdk'
2
+ require 'toiler/utils/environment_loader'
3
+ require 'toiler/utils/logging'
4
+ require 'toiler/utils/argument_parser'
5
+ require 'toiler/worker'
6
+ require 'toiler/cli'
7
+ require 'toiler/version'
8
+
9
+ # Main module
10
+ module Toiler
11
+ @worker_class_registry = {}
12
+ @options = {
13
+ aws: {}
14
+ }
15
+ @fetchers = {}
16
+ @processor_pools = {}
17
+
18
+ attr_reader :worker_class_registry, :options, :fetchers, :processor_pools
19
+ module_function :worker_class_registry, :options, :fetchers, :processor_pools
20
+
21
+ module_function
22
+
23
+ def logger
24
+ Toiler::Utils::Logging.logger
25
+ end
26
+
27
+ def queues
28
+ worker_class_registry.keys
29
+ end
30
+
31
+ def fetcher(queue)
32
+ fetchers["fetcher_#{queue}".to_sym]
33
+ end
34
+
35
+ def set_fetcher(queue, val)
36
+ fetchers["fetcher_#{queue}".to_sym] = val
37
+ end
38
+
39
+ def processor_pool(queue)
40
+ processor_pools["processor_pool_#{queue}".to_sym]
41
+ end
42
+
43
+ def set_processor_pool(queue, val)
44
+ processor_pools["processor_pool_#{queue}".to_sym] = val
45
+ end
46
+
47
+ def default_options
48
+ {
49
+ auto_visibility_timeout: false,
50
+ concurrency: 1,
51
+ auto_delete: false,
52
+ batch: false
53
+ }
54
+ end
55
+ end
data/toiler.gemspec CHANGED
@@ -1,30 +1,30 @@
1
- # coding: utf-8
2
- lib = File.expand_path('../lib', __FILE__)
3
- $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
- require 'toiler/version'
5
-
6
- Gem::Specification.new do |spec|
7
- spec.name = 'toiler'
8
- spec.version = Toiler::VERSION
9
- spec.authors = ['Sebastian Schepens']
10
- spec.email = ['sebas.schep@hotmail.com']
11
- spec.description = spec.summary = 'Toiler is a super efficient AWS SQS thread based message processor'
12
- spec.homepage = 'https://github.com/sschepens/toiler'
13
- spec.license = 'LGPLv3'
14
-
15
- spec.files = `git ls-files -z`.split("\x0")
16
- spec.executables << 'toiler'
17
- spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
- spec.require_paths = ['lib']
19
-
20
- spec.add_development_dependency 'bundler', '~> 1.6'
21
- spec.add_development_dependency 'rake'
22
- spec.add_development_dependency 'rspec'
23
- spec.add_development_dependency 'pry-byebug'
24
- spec.add_development_dependency 'nokogiri'
25
- spec.add_development_dependency 'dotenv'
26
-
27
- spec.add_dependency 'aws-sdk', '~> 2.2', '>= 2.2.10'
28
- spec.add_dependency 'concurrent-ruby', '~> 1.0', '>= 1.0.0'
29
- spec.add_dependency 'concurrent-ruby-edge', '~> 0.2.0', '>= 0.2.0'
30
- end
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'toiler/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'toiler'
8
+ spec.version = Toiler::VERSION
9
+ spec.authors = ['Sebastian Schepens']
10
+ spec.email = ['sebas.schep@hotmail.com']
11
+ spec.description = spec.summary = 'Toiler is a super efficient AWS SQS thread based message processor'
12
+ spec.homepage = 'https://github.com/sschepens/toiler'
13
+ spec.license = 'LGPLv3'
14
+
15
+ spec.files = `git ls-files -z`.split("\x0")
16
+ spec.executables << 'toiler'
17
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
+ spec.require_paths = ['lib']
19
+
20
+ spec.add_development_dependency 'bundler', '~> 1.6'
21
+ spec.add_development_dependency 'rake'
22
+ spec.add_development_dependency 'rspec'
23
+ spec.add_development_dependency 'pry-byebug'
24
+ spec.add_development_dependency 'nokogiri'
25
+ spec.add_development_dependency 'dotenv'
26
+
27
+ spec.add_dependency 'aws-sdk', '~> 2.2', '>= 2.2.10'
28
+ spec.add_dependency 'concurrent-ruby', '~> 1.0', '>= 1.0.0'
29
+ spec.add_dependency 'concurrent-ruby-edge', '~> 0.2.0', '>= 0.2.0'
30
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: toiler
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.1.beta1
4
+ version: 0.3.1.beta2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sebastian Schepens