toiler 0.1.5 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ruby-version +1 -0
- data/Gemfile +7 -4
- data/Gemfile.lock +26 -57
- data/README.md +15 -0
- data/lib/toiler.rb +15 -44
- data/lib/toiler/actor/fetcher.rb +89 -0
- data/lib/toiler/actor/processor.rb +106 -0
- data/lib/toiler/actor/supervisor.rb +45 -0
- data/lib/toiler/actor/utils/actor_logging.rb +28 -0
- data/lib/toiler/aws/message.rb +64 -0
- data/lib/toiler/aws/queue.rb +61 -0
- data/lib/toiler/cli.rb +67 -94
- data/lib/toiler/utils/argument_parser.rb +50 -0
- data/lib/toiler/utils/environment_loader.rb +104 -0
- data/lib/toiler/utils/logging.rb +37 -0
- data/lib/toiler/version.rb +2 -1
- data/lib/toiler/worker.rb +35 -15
- data/toiler.gemspec +4 -4
- metadata +32 -32
- data/celluloid-task-pooledfiber/.gitignore +0 -9
- data/celluloid-task-pooledfiber/.rspec +0 -2
- data/celluloid-task-pooledfiber/.travis.yml +0 -5
- data/celluloid-task-pooledfiber/Gemfile +0 -11
- data/celluloid-task-pooledfiber/LICENSE.txt +0 -21
- data/celluloid-task-pooledfiber/README.md +0 -37
- data/celluloid-task-pooledfiber/Rakefile +0 -8
- data/celluloid-task-pooledfiber/celluloid-task-pooled-fiber.gemspec +0 -18
- data/celluloid-task-pooledfiber/lib/celluloid/task/pooled_fiber.rb +0 -26
- data/celluloid-task-pooledfiber/lib/celluloid/util/fiber_pool.rb +0 -95
- data/celluloid-task-pooledfiber/spec/celluloid/tasks/pooled_fiber_spec.rb +0 -5
- data/celluloid-task-pooledfiber/spec/spec_helper.rb +0 -60
- data/celluloid-task-pooledfiber/spec/support/shared_examples_for_task.rb +0 -49
- data/lib/toiler/core_ext.rb +0 -47
- data/lib/toiler/environment_loader.rb +0 -82
- data/lib/toiler/fetcher.rb +0 -56
- data/lib/toiler/logging.rb +0 -42
- data/lib/toiler/manager.rb +0 -71
- data/lib/toiler/message.rb +0 -60
- data/lib/toiler/processor.rb +0 -86
- data/lib/toiler/queue.rb +0 -53
- data/lib/toiler/scheduler.rb +0 -16
- data/lib/toiler/supervisor.rb +0 -66
@@ -0,0 +1,104 @@
|
|
1
|
+
require 'erb'
|
2
|
+
require 'yaml'
|
3
|
+
|
4
|
+
module Toiler
|
5
|
+
module Utils
|
6
|
+
# Takes care of loading componentes to get toiler ready to run
|
7
|
+
class EnvironmentLoader
|
8
|
+
attr_reader :options
|
9
|
+
|
10
|
+
def self.load(options)
|
11
|
+
new(options).load
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.load_for_rails_console
|
15
|
+
load(config_file: (Rails.root + 'config' + 'toiler.yml'))
|
16
|
+
end
|
17
|
+
|
18
|
+
def initialize(options)
|
19
|
+
@options = options
|
20
|
+
end
|
21
|
+
|
22
|
+
def load
|
23
|
+
initialize_logger
|
24
|
+
load_rails if options[:rails]
|
25
|
+
require_workers if options[:require]
|
26
|
+
Toiler.options.merge!(config_file_options)
|
27
|
+
Toiler.options.merge!(options)
|
28
|
+
initialize_aws
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def config_file_options
|
34
|
+
if (path = options[:config_file])
|
35
|
+
unless File.exist?(path)
|
36
|
+
Toiler.logger.warn "Config file #{path} does not exist"
|
37
|
+
path = nil
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
return {} unless path
|
42
|
+
|
43
|
+
deep_symbolize_keys YAML.load(ERB.new(File.read(path)).result)
|
44
|
+
end
|
45
|
+
|
46
|
+
def initialize_aws
|
47
|
+
return if Toiler.options[:aws].empty?
|
48
|
+
::Aws.config[:region] = Toiler.options[:aws][:region]
|
49
|
+
set_aws_credentials
|
50
|
+
end
|
51
|
+
|
52
|
+
def set_aws_credentials
|
53
|
+
return unless Toiler.options[:aws][:access_key_id]
|
54
|
+
::Aws.config[:credentials] = ::Aws::Credentials.new(
|
55
|
+
Toiler.options[:aws][:access_key_id],
|
56
|
+
Toiler.options[:aws][:secret_access_key]
|
57
|
+
)
|
58
|
+
end
|
59
|
+
|
60
|
+
def initialize_logger
|
61
|
+
Toiler::Utils::Logging.initialize_logger(options[:logfile])
|
62
|
+
Toiler.logger.level = Logger::DEBUG if options[:verbose]
|
63
|
+
end
|
64
|
+
|
65
|
+
def load_rails
|
66
|
+
require 'rails'
|
67
|
+
if ::Rails::VERSION::MAJOR < 4
|
68
|
+
load_rails_old
|
69
|
+
else
|
70
|
+
load_rails_new
|
71
|
+
end
|
72
|
+
Toiler.logger.info 'Rails environment loaded'
|
73
|
+
end
|
74
|
+
|
75
|
+
def load_rails_old
|
76
|
+
require File.expand_path('config/environment.rb')
|
77
|
+
::Rails.application.eager_load!
|
78
|
+
end
|
79
|
+
|
80
|
+
def load_rails_new
|
81
|
+
require File.expand_path('config/application.rb')
|
82
|
+
::Rails::Application.initializer 'toiler.eager_load' do
|
83
|
+
::Rails.application.config.eager_load = true
|
84
|
+
end
|
85
|
+
require File.expand_path('config/environment.rb')
|
86
|
+
end
|
87
|
+
|
88
|
+
def require_workers
|
89
|
+
require options[:require]
|
90
|
+
end
|
91
|
+
|
92
|
+
def deep_symbolize_keys(h)
|
93
|
+
h.each_with_object({}) do |(key, value), result|
|
94
|
+
k = key.respond_to?(:to_sym) ? key.to_sym : key
|
95
|
+
result[k] = if value.is_a? Hash
|
96
|
+
deep_symbolize_keys value
|
97
|
+
else
|
98
|
+
value
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'time'
|
2
|
+
require 'logger'
|
3
|
+
|
4
|
+
module Toiler
|
5
|
+
module Utils
|
6
|
+
# Initializes and exposes Toiler's default logger
|
7
|
+
module Logging
|
8
|
+
# Toiler's default log formatter
|
9
|
+
class Pretty < Logger::Formatter
|
10
|
+
def call(sev, time, progname, msg)
|
11
|
+
formatted = msg.respond_to?(:gsub) ? msg.gsub("\n", "\n\t") : msg
|
12
|
+
time = time.utc.iso8601
|
13
|
+
pid = Process.pid
|
14
|
+
"#{time} Pid:#{pid} Actor:#{progname} Level:#{sev}: #{formatted}\n"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
module_function
|
19
|
+
|
20
|
+
def initialize_logger(log_target = STDOUT)
|
21
|
+
log_target = STDOUT if log_target.nil?
|
22
|
+
@logger = Logger.new(log_target)
|
23
|
+
@logger.level = Logger::INFO
|
24
|
+
@logger.formatter = Pretty.new
|
25
|
+
@logger
|
26
|
+
end
|
27
|
+
|
28
|
+
def logger
|
29
|
+
@logger || initialize_logger
|
30
|
+
end
|
31
|
+
|
32
|
+
def logger=(log)
|
33
|
+
@logger = (log ? log : Logger.new('/dev/null'))
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/lib/toiler/version.rb
CHANGED
data/lib/toiler/worker.rb
CHANGED
@@ -1,41 +1,61 @@
|
|
1
1
|
module Toiler
|
2
|
+
# Toiler's Worker behaviour
|
2
3
|
module Worker
|
3
4
|
def self.included(base)
|
4
5
|
base.extend(ClassMethods)
|
6
|
+
base.class_variable_set(:@@toiler_options, Toiler.default_options)
|
5
7
|
end
|
6
8
|
|
9
|
+
def log(level, message)
|
10
|
+
Toiler.logger.log(level, message, self.class)
|
11
|
+
end
|
12
|
+
|
13
|
+
def error(msg)
|
14
|
+
log Logger::Severity::ERROR, msg
|
15
|
+
end
|
16
|
+
|
17
|
+
def info(msg)
|
18
|
+
log Logger::Severity::INFO, msg
|
19
|
+
end
|
20
|
+
|
21
|
+
def debug(msg)
|
22
|
+
log Logger::Severity::DEBUG, msg
|
23
|
+
end
|
24
|
+
|
25
|
+
def warn(msg)
|
26
|
+
log Logger::Severity::WARN, msg
|
27
|
+
end
|
28
|
+
|
29
|
+
def fatal(msg)
|
30
|
+
log Logger::Severity::FATAL, msg
|
31
|
+
end
|
32
|
+
|
33
|
+
# Class methods for Workers
|
7
34
|
module ClassMethods
|
8
|
-
def toiler_options(options)
|
9
|
-
if
|
10
|
-
@toiler_options = @toiler_options.merge options
|
11
|
-
else
|
12
|
-
@toiler_options = Toiler.default_options.merge options
|
13
|
-
end
|
35
|
+
def toiler_options(options = {})
|
36
|
+
return class_variable_get(:@@toiler_options) if options.empty?
|
14
37
|
Toiler.worker_class_registry[options[:queue]] = self if options[:queue]
|
15
|
-
|
16
|
-
|
17
|
-
def get_toiler_options
|
18
|
-
@toiler_options
|
38
|
+
class_variable_get(:@@toiler_options).merge! options
|
19
39
|
end
|
20
40
|
|
21
41
|
def batch?
|
22
|
-
|
42
|
+
class_variable_get(:@@toiler_options)[:batch]
|
23
43
|
end
|
24
44
|
|
25
45
|
def concurrency
|
26
|
-
|
46
|
+
class_variable_get(:@@toiler_options)[:concurrency]
|
27
47
|
end
|
28
48
|
|
29
49
|
def queue
|
30
|
-
|
50
|
+
class_variable_get(:@@toiler_options)[:queue]
|
31
51
|
end
|
32
52
|
|
33
53
|
def auto_visibility_timeout?
|
34
|
-
|
54
|
+
class_variable_get(:@@toiler_options)[:auto_visibility_timeout]
|
35
55
|
end
|
36
56
|
|
37
57
|
def auto_delete?
|
38
|
-
|
58
|
+
class_variable_get(:@@toiler_options)[:auto_delete]
|
39
59
|
end
|
40
60
|
end
|
41
61
|
end
|
data/toiler.gemspec
CHANGED
@@ -13,10 +13,9 @@ Gem::Specification.new do |spec|
|
|
13
13
|
spec.license = 'LGPLv3'
|
14
14
|
|
15
15
|
spec.files = `git ls-files -z`.split("\x0")
|
16
|
-
spec.files += `cd celluloid-task-pooledfiber && git ls-files -z`.split("\x0").collect { |f| "celluloid-task-pooledfiber/#{f}" }
|
17
16
|
spec.executables << 'toiler'
|
18
17
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
|
-
spec.require_paths = ['lib'
|
18
|
+
spec.require_paths = ['lib']
|
20
19
|
|
21
20
|
spec.add_development_dependency 'bundler', '~> 1.6'
|
22
21
|
spec.add_development_dependency 'rake'
|
@@ -25,6 +24,7 @@ Gem::Specification.new do |spec|
|
|
25
24
|
spec.add_development_dependency 'nokogiri'
|
26
25
|
spec.add_development_dependency 'dotenv'
|
27
26
|
|
28
|
-
spec.add_dependency 'aws-sdk', '~> 2.
|
29
|
-
spec.add_dependency '
|
27
|
+
spec.add_dependency 'aws-sdk', '~> 2.1.16'
|
28
|
+
spec.add_dependency 'concurrent-ruby', '~> 0.9.0'
|
29
|
+
spec.add_dependency 'concurrent-ruby-edge', '~> 0.1.0'
|
30
30
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: toiler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sebastian Schepens
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-08-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -100,28 +100,42 @@ dependencies:
|
|
100
100
|
requirements:
|
101
101
|
- - "~>"
|
102
102
|
- !ruby/object:Gem::Version
|
103
|
-
version: 2.
|
103
|
+
version: 2.1.16
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
108
|
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
|
-
version: 2.
|
110
|
+
version: 2.1.16
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
|
-
name:
|
112
|
+
name: concurrent-ruby
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
115
|
- - "~>"
|
116
116
|
- !ruby/object:Gem::Version
|
117
|
-
version: 0.
|
117
|
+
version: 0.9.0
|
118
118
|
type: :runtime
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
122
|
- - "~>"
|
123
123
|
- !ruby/object:Gem::Version
|
124
|
-
version: 0.
|
124
|
+
version: 0.9.0
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: concurrent-ruby-edge
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - "~>"
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: 0.1.0
|
132
|
+
type: :runtime
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - "~>"
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: 0.1.0
|
125
139
|
description: Toiler is a super efficient AWS SQS thread based message processor
|
126
140
|
email:
|
127
141
|
- sebas.schep@hotmail.com
|
@@ -132,37 +146,24 @@ extra_rdoc_files: []
|
|
132
146
|
files:
|
133
147
|
- ".gitignore"
|
134
148
|
- ".gitmodules"
|
149
|
+
- ".ruby-version"
|
135
150
|
- Gemfile
|
136
151
|
- Gemfile.lock
|
137
152
|
- LICENSE
|
138
153
|
- README.md
|
139
154
|
- Rakefile
|
140
155
|
- bin/toiler
|
141
|
-
- celluloid-task-pooledfiber/.gitignore
|
142
|
-
- celluloid-task-pooledfiber/.rspec
|
143
|
-
- celluloid-task-pooledfiber/.travis.yml
|
144
|
-
- celluloid-task-pooledfiber/Gemfile
|
145
|
-
- celluloid-task-pooledfiber/LICENSE.txt
|
146
|
-
- celluloid-task-pooledfiber/README.md
|
147
|
-
- celluloid-task-pooledfiber/Rakefile
|
148
|
-
- celluloid-task-pooledfiber/celluloid-task-pooled-fiber.gemspec
|
149
|
-
- celluloid-task-pooledfiber/lib/celluloid/task/pooled_fiber.rb
|
150
|
-
- celluloid-task-pooledfiber/lib/celluloid/util/fiber_pool.rb
|
151
|
-
- celluloid-task-pooledfiber/spec/celluloid/tasks/pooled_fiber_spec.rb
|
152
|
-
- celluloid-task-pooledfiber/spec/spec_helper.rb
|
153
|
-
- celluloid-task-pooledfiber/spec/support/shared_examples_for_task.rb
|
154
156
|
- lib/toiler.rb
|
157
|
+
- lib/toiler/actor/fetcher.rb
|
158
|
+
- lib/toiler/actor/processor.rb
|
159
|
+
- lib/toiler/actor/supervisor.rb
|
160
|
+
- lib/toiler/actor/utils/actor_logging.rb
|
161
|
+
- lib/toiler/aws/message.rb
|
162
|
+
- lib/toiler/aws/queue.rb
|
155
163
|
- lib/toiler/cli.rb
|
156
|
-
- lib/toiler/
|
157
|
-
- lib/toiler/environment_loader.rb
|
158
|
-
- lib/toiler/
|
159
|
-
- lib/toiler/logging.rb
|
160
|
-
- lib/toiler/manager.rb
|
161
|
-
- lib/toiler/message.rb
|
162
|
-
- lib/toiler/processor.rb
|
163
|
-
- lib/toiler/queue.rb
|
164
|
-
- lib/toiler/scheduler.rb
|
165
|
-
- lib/toiler/supervisor.rb
|
164
|
+
- lib/toiler/utils/argument_parser.rb
|
165
|
+
- lib/toiler/utils/environment_loader.rb
|
166
|
+
- lib/toiler/utils/logging.rb
|
166
167
|
- lib/toiler/version.rb
|
167
168
|
- lib/toiler/worker.rb
|
168
169
|
- toiler.gemspec
|
@@ -174,7 +175,6 @@ post_install_message:
|
|
174
175
|
rdoc_options: []
|
175
176
|
require_paths:
|
176
177
|
- lib
|
177
|
-
- celluloid-task-pooledfiber/lib
|
178
178
|
required_ruby_version: !ruby/object:Gem::Requirement
|
179
179
|
requirements:
|
180
180
|
- - ">="
|
@@ -187,7 +187,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
187
187
|
version: '0'
|
188
188
|
requirements: []
|
189
189
|
rubyforge_project:
|
190
|
-
rubygems_version: 2.4.
|
190
|
+
rubygems_version: 2.4.8
|
191
191
|
signing_key:
|
192
192
|
specification_version: 4
|
193
193
|
summary: Toiler is a super efficient AWS SQS thread based message processor
|
@@ -1,21 +0,0 @@
|
|
1
|
-
The MIT License (MIT)
|
2
|
-
|
3
|
-
Copyright (c) 2015 Chris Heald
|
4
|
-
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
7
|
-
in the Software without restriction, including without limitation the rights
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
10
|
-
furnished to do so, subject to the following conditions:
|
11
|
-
|
12
|
-
The above copyright notice and this permission notice shall be included in
|
13
|
-
all copies or substantial portions of the Software.
|
14
|
-
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
-
THE SOFTWARE.
|
@@ -1,37 +0,0 @@
|
|
1
|
-
# Celluloid::Task::PooledFiber
|
2
|
-
|
3
|
-
[](https://travis-ci.org/celluloid/celluloid-task-pooledfiber)
|
4
|
-
|
5
|
-
Provides a Task implementation which reuses a pool of fibers to run tasks. This improves Celluloid's performance.
|
6
|
-
|
7
|
-
## Installation
|
8
|
-
|
9
|
-
Add this line to your application's Gemfile:
|
10
|
-
|
11
|
-
```ruby
|
12
|
-
gem 'celluloid-task-pooledfiber'
|
13
|
-
```
|
14
|
-
|
15
|
-
And then execute:
|
16
|
-
|
17
|
-
$ bundle
|
18
|
-
|
19
|
-
Or install it yourself as:
|
20
|
-
|
21
|
-
$ gem install celluloid-task-pooledfiber
|
22
|
-
|
23
|
-
## Usage
|
24
|
-
|
25
|
-
In your application, before starting Celluloid:
|
26
|
-
|
27
|
-
Celluloid.task_class = Celluloid::Task::PooledFiber
|
28
|
-
|
29
|
-
That's it.
|
30
|
-
|
31
|
-
## Contributing
|
32
|
-
|
33
|
-
1. Fork it ( https://github.com/cheald/celluloid-task-pooledfiber/fork )
|
34
|
-
2. Create your feature branch (`git checkout -b my-new-feature`)
|
35
|
-
3. Commit your changes (`git commit -am 'Add some feature'`)
|
36
|
-
4. Push to the branch (`git push origin my-new-feature`)
|
37
|
-
5. Create a new Pull Request
|