toiler 0.1.5 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/.ruby-version +1 -0
  3. data/Gemfile +7 -4
  4. data/Gemfile.lock +26 -57
  5. data/README.md +15 -0
  6. data/lib/toiler.rb +15 -44
  7. data/lib/toiler/actor/fetcher.rb +89 -0
  8. data/lib/toiler/actor/processor.rb +106 -0
  9. data/lib/toiler/actor/supervisor.rb +45 -0
  10. data/lib/toiler/actor/utils/actor_logging.rb +28 -0
  11. data/lib/toiler/aws/message.rb +64 -0
  12. data/lib/toiler/aws/queue.rb +61 -0
  13. data/lib/toiler/cli.rb +67 -94
  14. data/lib/toiler/utils/argument_parser.rb +50 -0
  15. data/lib/toiler/utils/environment_loader.rb +104 -0
  16. data/lib/toiler/utils/logging.rb +37 -0
  17. data/lib/toiler/version.rb +2 -1
  18. data/lib/toiler/worker.rb +35 -15
  19. data/toiler.gemspec +4 -4
  20. metadata +32 -32
  21. data/celluloid-task-pooledfiber/.gitignore +0 -9
  22. data/celluloid-task-pooledfiber/.rspec +0 -2
  23. data/celluloid-task-pooledfiber/.travis.yml +0 -5
  24. data/celluloid-task-pooledfiber/Gemfile +0 -11
  25. data/celluloid-task-pooledfiber/LICENSE.txt +0 -21
  26. data/celluloid-task-pooledfiber/README.md +0 -37
  27. data/celluloid-task-pooledfiber/Rakefile +0 -8
  28. data/celluloid-task-pooledfiber/celluloid-task-pooled-fiber.gemspec +0 -18
  29. data/celluloid-task-pooledfiber/lib/celluloid/task/pooled_fiber.rb +0 -26
  30. data/celluloid-task-pooledfiber/lib/celluloid/util/fiber_pool.rb +0 -95
  31. data/celluloid-task-pooledfiber/spec/celluloid/tasks/pooled_fiber_spec.rb +0 -5
  32. data/celluloid-task-pooledfiber/spec/spec_helper.rb +0 -60
  33. data/celluloid-task-pooledfiber/spec/support/shared_examples_for_task.rb +0 -49
  34. data/lib/toiler/core_ext.rb +0 -47
  35. data/lib/toiler/environment_loader.rb +0 -82
  36. data/lib/toiler/fetcher.rb +0 -56
  37. data/lib/toiler/logging.rb +0 -42
  38. data/lib/toiler/manager.rb +0 -71
  39. data/lib/toiler/message.rb +0 -60
  40. data/lib/toiler/processor.rb +0 -86
  41. data/lib/toiler/queue.rb +0 -53
  42. data/lib/toiler/scheduler.rb +0 -16
  43. data/lib/toiler/supervisor.rb +0 -66
@@ -0,0 +1,104 @@
1
+ require 'erb'
2
+ require 'yaml'
3
+
4
+ module Toiler
5
+ module Utils
6
+ # Takes care of loading componentes to get toiler ready to run
7
+ class EnvironmentLoader
8
+ attr_reader :options
9
+
10
+ def self.load(options)
11
+ new(options).load
12
+ end
13
+
14
+ def self.load_for_rails_console
15
+ load(config_file: (Rails.root + 'config' + 'toiler.yml'))
16
+ end
17
+
18
+ def initialize(options)
19
+ @options = options
20
+ end
21
+
22
+ def load
23
+ initialize_logger
24
+ load_rails if options[:rails]
25
+ require_workers if options[:require]
26
+ Toiler.options.merge!(config_file_options)
27
+ Toiler.options.merge!(options)
28
+ initialize_aws
29
+ end
30
+
31
+ private
32
+
33
+ def config_file_options
34
+ if (path = options[:config_file])
35
+ unless File.exist?(path)
36
+ Toiler.logger.warn "Config file #{path} does not exist"
37
+ path = nil
38
+ end
39
+ end
40
+
41
+ return {} unless path
42
+
43
+ deep_symbolize_keys YAML.load(ERB.new(File.read(path)).result)
44
+ end
45
+
46
+ def initialize_aws
47
+ return if Toiler.options[:aws].empty?
48
+ ::Aws.config[:region] = Toiler.options[:aws][:region]
49
+ set_aws_credentials
50
+ end
51
+
52
+ def set_aws_credentials
53
+ return unless Toiler.options[:aws][:access_key_id]
54
+ ::Aws.config[:credentials] = ::Aws::Credentials.new(
55
+ Toiler.options[:aws][:access_key_id],
56
+ Toiler.options[:aws][:secret_access_key]
57
+ )
58
+ end
59
+
60
+ def initialize_logger
61
+ Toiler::Utils::Logging.initialize_logger(options[:logfile])
62
+ Toiler.logger.level = Logger::DEBUG if options[:verbose]
63
+ end
64
+
65
+ def load_rails
66
+ require 'rails'
67
+ if ::Rails::VERSION::MAJOR < 4
68
+ load_rails_old
69
+ else
70
+ load_rails_new
71
+ end
72
+ Toiler.logger.info 'Rails environment loaded'
73
+ end
74
+
75
+ def load_rails_old
76
+ require File.expand_path('config/environment.rb')
77
+ ::Rails.application.eager_load!
78
+ end
79
+
80
+ def load_rails_new
81
+ require File.expand_path('config/application.rb')
82
+ ::Rails::Application.initializer 'toiler.eager_load' do
83
+ ::Rails.application.config.eager_load = true
84
+ end
85
+ require File.expand_path('config/environment.rb')
86
+ end
87
+
88
+ def require_workers
89
+ require options[:require]
90
+ end
91
+
92
+ def deep_symbolize_keys(h)
93
+ h.each_with_object({}) do |(key, value), result|
94
+ k = key.respond_to?(:to_sym) ? key.to_sym : key
95
+ result[k] = if value.is_a? Hash
96
+ deep_symbolize_keys value
97
+ else
98
+ value
99
+ end
100
+ end
101
+ end
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,37 @@
1
+ require 'time'
2
+ require 'logger'
3
+
4
+ module Toiler
5
+ module Utils
6
+ # Initializes and exposes Toiler's default logger
7
+ module Logging
8
+ # Toiler's default log formatter
9
+ class Pretty < Logger::Formatter
10
+ def call(sev, time, progname, msg)
11
+ formatted = msg.respond_to?(:gsub) ? msg.gsub("\n", "\n\t") : msg
12
+ time = time.utc.iso8601
13
+ pid = Process.pid
14
+ "#{time} Pid:#{pid} Actor:#{progname} Level:#{sev}: #{formatted}\n"
15
+ end
16
+ end
17
+
18
+ module_function
19
+
20
+ def initialize_logger(log_target = STDOUT)
21
+ log_target = STDOUT if log_target.nil?
22
+ @logger = Logger.new(log_target)
23
+ @logger.level = Logger::INFO
24
+ @logger.formatter = Pretty.new
25
+ @logger
26
+ end
27
+
28
+ def logger
29
+ @logger || initialize_logger
30
+ end
31
+
32
+ def logger=(log)
33
+ @logger = (log ? log : Logger.new('/dev/null'))
34
+ end
35
+ end
36
+ end
37
+ end
@@ -1,3 +1,4 @@
1
+ # Toiler Version
1
2
  module Toiler
2
- VERSION = '0.1.5'
3
+ VERSION = '0.2.0'
3
4
  end
data/lib/toiler/worker.rb CHANGED
@@ -1,41 +1,61 @@
1
1
  module Toiler
2
+ # Toiler's Worker behaviour
2
3
  module Worker
3
4
  def self.included(base)
4
5
  base.extend(ClassMethods)
6
+ base.class_variable_set(:@@toiler_options, Toiler.default_options)
5
7
  end
6
8
 
9
+ def log(level, message)
10
+ Toiler.logger.log(level, message, self.class)
11
+ end
12
+
13
+ def error(msg)
14
+ log Logger::Severity::ERROR, msg
15
+ end
16
+
17
+ def info(msg)
18
+ log Logger::Severity::INFO, msg
19
+ end
20
+
21
+ def debug(msg)
22
+ log Logger::Severity::DEBUG, msg
23
+ end
24
+
25
+ def warn(msg)
26
+ log Logger::Severity::WARN, msg
27
+ end
28
+
29
+ def fatal(msg)
30
+ log Logger::Severity::FATAL, msg
31
+ end
32
+
33
+ # Class methods for Workers
7
34
  module ClassMethods
8
- def toiler_options(options)
9
- if @toiler_options
10
- @toiler_options = @toiler_options.merge options
11
- else
12
- @toiler_options = Toiler.default_options.merge options
13
- end
35
+ def toiler_options(options = {})
36
+ return class_variable_get(:@@toiler_options) if options.empty?
14
37
  Toiler.worker_class_registry[options[:queue]] = self if options[:queue]
15
- end
16
-
17
- def get_toiler_options
18
- @toiler_options
38
+ class_variable_get(:@@toiler_options).merge! options
19
39
  end
20
40
 
21
41
  def batch?
22
- @toiler_options[:batch]
42
+ class_variable_get(:@@toiler_options)[:batch]
23
43
  end
24
44
 
25
45
  def concurrency
26
- @toiler_options[:concurrency]
46
+ class_variable_get(:@@toiler_options)[:concurrency]
27
47
  end
28
48
 
29
49
  def queue
30
- @toiler_options[:queue]
50
+ class_variable_get(:@@toiler_options)[:queue]
31
51
  end
32
52
 
33
53
  def auto_visibility_timeout?
34
- @toiler_options[:auto_visibility_timeout]
54
+ class_variable_get(:@@toiler_options)[:auto_visibility_timeout]
35
55
  end
36
56
 
37
57
  def auto_delete?
38
- @toiler_options[:auto_delete]
58
+ class_variable_get(:@@toiler_options)[:auto_delete]
39
59
  end
40
60
  end
41
61
  end
data/toiler.gemspec CHANGED
@@ -13,10 +13,9 @@ Gem::Specification.new do |spec|
13
13
  spec.license = 'LGPLv3'
14
14
 
15
15
  spec.files = `git ls-files -z`.split("\x0")
16
- spec.files += `cd celluloid-task-pooledfiber && git ls-files -z`.split("\x0").collect { |f| "celluloid-task-pooledfiber/#{f}" }
17
16
  spec.executables << 'toiler'
18
17
  spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
- spec.require_paths = ['lib', 'celluloid-task-pooledfiber/lib']
18
+ spec.require_paths = ['lib']
20
19
 
21
20
  spec.add_development_dependency 'bundler', '~> 1.6'
22
21
  spec.add_development_dependency 'rake'
@@ -25,6 +24,7 @@ Gem::Specification.new do |spec|
25
24
  spec.add_development_dependency 'nokogiri'
26
25
  spec.add_development_dependency 'dotenv'
27
26
 
28
- spec.add_dependency 'aws-sdk', '~> 2.0.21'
29
- spec.add_dependency 'celluloid', '~> 0.17.pre15'
27
+ spec.add_dependency 'aws-sdk', '~> 2.1.16'
28
+ spec.add_dependency 'concurrent-ruby', '~> 0.9.0'
29
+ spec.add_dependency 'concurrent-ruby-edge', '~> 0.1.0'
30
30
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: toiler
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.5
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sebastian Schepens
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-05-14 00:00:00.000000000 Z
11
+ date: 2015-08-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -100,28 +100,42 @@ dependencies:
100
100
  requirements:
101
101
  - - "~>"
102
102
  - !ruby/object:Gem::Version
103
- version: 2.0.21
103
+ version: 2.1.16
104
104
  type: :runtime
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
108
  - - "~>"
109
109
  - !ruby/object:Gem::Version
110
- version: 2.0.21
110
+ version: 2.1.16
111
111
  - !ruby/object:Gem::Dependency
112
- name: celluloid
112
+ name: concurrent-ruby
113
113
  requirement: !ruby/object:Gem::Requirement
114
114
  requirements:
115
115
  - - "~>"
116
116
  - !ruby/object:Gem::Version
117
- version: 0.17.pre15
117
+ version: 0.9.0
118
118
  type: :runtime
119
119
  prerelease: false
120
120
  version_requirements: !ruby/object:Gem::Requirement
121
121
  requirements:
122
122
  - - "~>"
123
123
  - !ruby/object:Gem::Version
124
- version: 0.17.pre15
124
+ version: 0.9.0
125
+ - !ruby/object:Gem::Dependency
126
+ name: concurrent-ruby-edge
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: 0.1.0
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: 0.1.0
125
139
  description: Toiler is a super efficient AWS SQS thread based message processor
126
140
  email:
127
141
  - sebas.schep@hotmail.com
@@ -132,37 +146,24 @@ extra_rdoc_files: []
132
146
  files:
133
147
  - ".gitignore"
134
148
  - ".gitmodules"
149
+ - ".ruby-version"
135
150
  - Gemfile
136
151
  - Gemfile.lock
137
152
  - LICENSE
138
153
  - README.md
139
154
  - Rakefile
140
155
  - bin/toiler
141
- - celluloid-task-pooledfiber/.gitignore
142
- - celluloid-task-pooledfiber/.rspec
143
- - celluloid-task-pooledfiber/.travis.yml
144
- - celluloid-task-pooledfiber/Gemfile
145
- - celluloid-task-pooledfiber/LICENSE.txt
146
- - celluloid-task-pooledfiber/README.md
147
- - celluloid-task-pooledfiber/Rakefile
148
- - celluloid-task-pooledfiber/celluloid-task-pooled-fiber.gemspec
149
- - celluloid-task-pooledfiber/lib/celluloid/task/pooled_fiber.rb
150
- - celluloid-task-pooledfiber/lib/celluloid/util/fiber_pool.rb
151
- - celluloid-task-pooledfiber/spec/celluloid/tasks/pooled_fiber_spec.rb
152
- - celluloid-task-pooledfiber/spec/spec_helper.rb
153
- - celluloid-task-pooledfiber/spec/support/shared_examples_for_task.rb
154
156
  - lib/toiler.rb
157
+ - lib/toiler/actor/fetcher.rb
158
+ - lib/toiler/actor/processor.rb
159
+ - lib/toiler/actor/supervisor.rb
160
+ - lib/toiler/actor/utils/actor_logging.rb
161
+ - lib/toiler/aws/message.rb
162
+ - lib/toiler/aws/queue.rb
155
163
  - lib/toiler/cli.rb
156
- - lib/toiler/core_ext.rb
157
- - lib/toiler/environment_loader.rb
158
- - lib/toiler/fetcher.rb
159
- - lib/toiler/logging.rb
160
- - lib/toiler/manager.rb
161
- - lib/toiler/message.rb
162
- - lib/toiler/processor.rb
163
- - lib/toiler/queue.rb
164
- - lib/toiler/scheduler.rb
165
- - lib/toiler/supervisor.rb
164
+ - lib/toiler/utils/argument_parser.rb
165
+ - lib/toiler/utils/environment_loader.rb
166
+ - lib/toiler/utils/logging.rb
166
167
  - lib/toiler/version.rb
167
168
  - lib/toiler/worker.rb
168
169
  - toiler.gemspec
@@ -174,7 +175,6 @@ post_install_message:
174
175
  rdoc_options: []
175
176
  require_paths:
176
177
  - lib
177
- - celluloid-task-pooledfiber/lib
178
178
  required_ruby_version: !ruby/object:Gem::Requirement
179
179
  requirements:
180
180
  - - ">="
@@ -187,7 +187,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
187
187
  version: '0'
188
188
  requirements: []
189
189
  rubyforge_project:
190
- rubygems_version: 2.4.5
190
+ rubygems_version: 2.4.8
191
191
  signing_key:
192
192
  specification_version: 4
193
193
  summary: Toiler is a super efficient AWS SQS thread based message processor
@@ -1,9 +0,0 @@
1
- /.bundle/
2
- /.yardoc
3
- /Gemfile.lock
4
- /_yardoc/
5
- /coverage/
6
- /doc/
7
- /pkg/
8
- /spec/reports/
9
- /tmp/
@@ -1,2 +0,0 @@
1
- --format documentation
2
- --color
@@ -1,5 +0,0 @@
1
- language: ruby
2
- rvm:
3
- - 2.2.1
4
- - rbx-2
5
- - jruby
@@ -1,11 +0,0 @@
1
- source 'https://rubygems.org'
2
-
3
- gem 'celluloid', github: 'celluloid/celluloid', branch: '0.17.0-prerelease'
4
-
5
- gemspec
6
-
7
- group :test do
8
- gem 'rake'
9
- gem 'rspec', '~> 3.2'
10
- gem 'rspec-retry'
11
- end
@@ -1,21 +0,0 @@
1
- The MIT License (MIT)
2
-
3
- Copyright (c) 2015 Chris Heald
4
-
5
- Permission is hereby granted, free of charge, to any person obtaining a copy
6
- of this software and associated documentation files (the "Software"), to deal
7
- in the Software without restriction, including without limitation the rights
8
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
- copies of the Software, and to permit persons to whom the Software is
10
- furnished to do so, subject to the following conditions:
11
-
12
- The above copyright notice and this permission notice shall be included in
13
- all copies or substantial portions of the Software.
14
-
15
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
- THE SOFTWARE.
@@ -1,37 +0,0 @@
1
- # Celluloid::Task::PooledFiber
2
-
3
- [![Build Status](https://travis-ci.org/celluloid/celluloid-task-pooledfiber.svg)](https://travis-ci.org/celluloid/celluloid-task-pooledfiber)
4
-
5
- Provides a Task implementation which reuses a pool of fibers to run tasks. This improves Celluloid's performance.
6
-
7
- ## Installation
8
-
9
- Add this line to your application's Gemfile:
10
-
11
- ```ruby
12
- gem 'celluloid-task-pooledfiber'
13
- ```
14
-
15
- And then execute:
16
-
17
- $ bundle
18
-
19
- Or install it yourself as:
20
-
21
- $ gem install celluloid-task-pooledfiber
22
-
23
- ## Usage
24
-
25
- In your application, before starting Celluloid:
26
-
27
- Celluloid.task_class = Celluloid::Task::PooledFiber
28
-
29
- That's it.
30
-
31
- ## Contributing
32
-
33
- 1. Fork it ( https://github.com/cheald/celluloid-task-pooledfiber/fork )
34
- 2. Create your feature branch (`git checkout -b my-new-feature`)
35
- 3. Commit your changes (`git commit -am 'Add some feature'`)
36
- 4. Push to the branch (`git push origin my-new-feature`)
37
- 5. Create a new Pull Request