toiler 0.1.5 → 0.2.0

data/lib/toiler/utils/environment_loader.rb

@@ -0,0 +1,104 @@
+require 'erb'
+require 'yaml'
+
+module Toiler
+  module Utils
+    # Takes care of loading componentes to get toiler ready to run
+    class EnvironmentLoader
+      attr_reader :options
+
+      def self.load(options)
+        new(options).load
+      end
+
+      def self.load_for_rails_console
+        load(config_file: (Rails.root + 'config' + 'toiler.yml'))
+      end
+
+      def initialize(options)
+        @options = options
+      end
+
+      def load
+        initialize_logger
+        load_rails if options[:rails]
+        require_workers if options[:require]
+        Toiler.options.merge!(config_file_options)
+        Toiler.options.merge!(options)
+        initialize_aws
+      end
+
+      private
+
+      def config_file_options
+        if (path = options[:config_file])
+          unless File.exist?(path)
+            Toiler.logger.warn "Config file #{path} does not exist"
+            path = nil
+          end
+        end
+
+        return {} unless path
+
+        deep_symbolize_keys YAML.load(ERB.new(File.read(path)).result)
+      end
+
+      def initialize_aws
+        return if Toiler.options[:aws].empty?
+        ::Aws.config[:region] = Toiler.options[:aws][:region]
+        set_aws_credentials
+      end
+
+      def set_aws_credentials
+        return unless Toiler.options[:aws][:access_key_id]
+        ::Aws.config[:credentials] = ::Aws::Credentials.new(
+          Toiler.options[:aws][:access_key_id],
+          Toiler.options[:aws][:secret_access_key]
+        )
+      end
+
+      def initialize_logger
+        Toiler::Utils::Logging.initialize_logger(options[:logfile])
+        Toiler.logger.level = Logger::DEBUG if options[:verbose]
+      end
+
+      def load_rails
+        require 'rails'
+        if ::Rails::VERSION::MAJOR < 4
+          load_rails_old
+        else
+          load_rails_new
+        end
+        Toiler.logger.info 'Rails environment loaded'
+      end
+
+      def load_rails_old
+        require File.expand_path('config/environment.rb')
+        ::Rails.application.eager_load!
+      end
+
+      def load_rails_new
+        require File.expand_path('config/application.rb')
+        ::Rails::Application.initializer 'toiler.eager_load' do
+          ::Rails.application.config.eager_load = true
+        end
+        require File.expand_path('config/environment.rb')
+      end
+
+      def require_workers
+        require options[:require]
+      end
+
+      def deep_symbolize_keys(h)
+        h.each_with_object({}) do |(key, value), result|
+          k = key.respond_to?(:to_sym) ? key.to_sym : key
+          result[k] = if value.is_a? Hash
+                        deep_symbolize_keys value
+                      else
+                        value
+                      end
+        end
+      end
+    end
+  end
+end

data/lib/toiler/utils/logging.rb

@@ -0,0 +1,37 @@
+require 'time'
+require 'logger'
+
+module Toiler
+  module Utils
+    # Initializes and exposes Toiler's default logger
+    module Logging
+      # Toiler's default log formatter
+      class Pretty < Logger::Formatter
+        def call(sev, time, progname, msg)
+          formatted = msg.respond_to?(:gsub) ? msg.gsub("\n", "\n\t") : msg
+          time = time.utc.iso8601
+          pid = Process.pid
+          "#{time} Pid:#{pid} Actor:#{progname} Level:#{sev}: #{formatted}\n"
+        end
+      end
+
+      module_function
+
+      def initialize_logger(log_target = STDOUT)
+        log_target = STDOUT if log_target.nil?
+        @logger = Logger.new(log_target)
+        @logger.level = Logger::INFO
+        @logger.formatter = Pretty.new
+        @logger
+      end
+
+      def logger
+        @logger || initialize_logger
+      end
+
+      def logger=(log)
+        @logger = (log ? log : Logger.new('/dev/null'))
+      end
+    end
+  end
+end

data/lib/toiler/version.rb

@@ -1,3 +1,4 @@
+# Toiler Version
 module Toiler
-  VERSION = '0.1.5'
+  VERSION = '0.2.0'
 end

data/lib/toiler/worker.rb

@@ -1,41 +1,61 @@
 module Toiler
+  # Toiler's Worker behaviour
   module Worker
     def self.included(base)
       base.extend(ClassMethods)
+      base.class_variable_set(:@@toiler_options, Toiler.default_options)
     end
 
+    def log(level, message)
+      Toiler.logger.log(level, message, self.class)
+    end
+
+    def error(msg)
+      log Logger::Severity::ERROR, msg
+    end
+
+    def info(msg)
+      log Logger::Severity::INFO, msg
+    end
+
+    def debug(msg)
+      log Logger::Severity::DEBUG, msg
+    end
+
+    def warn(msg)
+      log Logger::Severity::WARN, msg
+    end
+
+    def fatal(msg)
+      log Logger::Severity::FATAL, msg
+    end
+
+    # Class methods for Workers
     module ClassMethods
-      def toiler_options(options)
-        if @toiler_options
-          @toiler_options = @toiler_options.merge options
-        else
-          @toiler_options = Toiler.default_options.merge options
-        end
+      def toiler_options(options = {})
+        return class_variable_get(:@@toiler_options) if options.empty?
         Toiler.worker_class_registry[options[:queue]] = self if options[:queue]
-      end
-
-      def get_toiler_options
-        @toiler_options
+        class_variable_get(:@@toiler_options).merge! options
       end
 
       def batch?
-        @toiler_options[:batch]
+        class_variable_get(:@@toiler_options)[:batch]
       end
 
       def concurrency
-        @toiler_options[:concurrency]
+        class_variable_get(:@@toiler_options)[:concurrency]
       end
 
       def queue
-        @toiler_options[:queue]
+        class_variable_get(:@@toiler_options)[:queue]
       end
 
       def auto_visibility_timeout?
-        @toiler_options[:auto_visibility_timeout]
+        class_variable_get(:@@toiler_options)[:auto_visibility_timeout]
       end
 
       def auto_delete?
-        @toiler_options[:auto_delete]
+        class_variable_get(:@@toiler_options)[:auto_delete]
       end
     end
   end

data/toiler.gemspec

@@ -13,10 +13,9 @@ Gem::Specification.new do |spec|
   spec.license       = 'LGPLv3'
 
   spec.files         = `git ls-files -z`.split("\x0")
-  spec.files += `cd celluloid-task-pooledfiber && git ls-files -z`.split("\x0").collect { |f| "celluloid-task-pooledfiber/#{f}" }
   spec.executables << 'toiler'
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ['lib', 'celluloid-task-pooledfiber/lib']
+  spec.require_paths = ['lib']
 
   spec.add_development_dependency 'bundler', '~> 1.6'
   spec.add_development_dependency 'rake'
@@ -25,6 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'nokogiri'
   spec.add_development_dependency 'dotenv'
 
-  spec.add_dependency 'aws-sdk', '~> 2.0.21'
-  spec.add_dependency 'celluloid', '~> 0.17.pre15'
+  spec.add_dependency 'aws-sdk', '~> 2.1.16'
+  spec.add_dependency 'concurrent-ruby', '~> 0.9.0'
+  spec.add_dependency 'concurrent-ruby-edge', '~> 0.1.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: toiler
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - Sebastian Schepens
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-14 00:00:00.000000000 Z
+date: 2015-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,28 +100,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.21
+        version: 2.1.16
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.21
+        version: 2.1.16
 - !ruby/object:Gem::Dependency
-  name: celluloid
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.17.pre15
+        version: 0.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.17.pre15
+        version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby-edge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 description: Toiler is a super efficient AWS SQS thread based message processor
 email:
 - sebas.schep@hotmail.com
@@ -132,37 +146,24 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".gitmodules"
+- ".ruby-version"
 - Gemfile
 - Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
 - bin/toiler
-- celluloid-task-pooledfiber/.gitignore
-- celluloid-task-pooledfiber/.rspec
-- celluloid-task-pooledfiber/.travis.yml
-- celluloid-task-pooledfiber/Gemfile
-- celluloid-task-pooledfiber/LICENSE.txt
-- celluloid-task-pooledfiber/README.md
-- celluloid-task-pooledfiber/Rakefile
-- celluloid-task-pooledfiber/celluloid-task-pooled-fiber.gemspec
-- celluloid-task-pooledfiber/lib/celluloid/task/pooled_fiber.rb
-- celluloid-task-pooledfiber/lib/celluloid/util/fiber_pool.rb
-- celluloid-task-pooledfiber/spec/celluloid/tasks/pooled_fiber_spec.rb
-- celluloid-task-pooledfiber/spec/spec_helper.rb
-- celluloid-task-pooledfiber/spec/support/shared_examples_for_task.rb
 - lib/toiler.rb
+- lib/toiler/actor/fetcher.rb
+- lib/toiler/actor/processor.rb
+- lib/toiler/actor/supervisor.rb
+- lib/toiler/actor/utils/actor_logging.rb
+- lib/toiler/aws/message.rb
+- lib/toiler/aws/queue.rb
 - lib/toiler/cli.rb
-- lib/toiler/core_ext.rb
-- lib/toiler/environment_loader.rb
-- lib/toiler/fetcher.rb
-- lib/toiler/logging.rb
-- lib/toiler/manager.rb
-- lib/toiler/message.rb
-- lib/toiler/processor.rb
-- lib/toiler/queue.rb
-- lib/toiler/scheduler.rb
-- lib/toiler/supervisor.rb
+- lib/toiler/utils/argument_parser.rb
+- lib/toiler/utils/environment_loader.rb
+- lib/toiler/utils/logging.rb
 - lib/toiler/version.rb
 - lib/toiler/worker.rb
 - toiler.gemspec
@@ -174,7 +175,6 @@ post_install_message:
 rdoc_options: []
 require_paths:
 - lib
-- celluloid-task-pooledfiber/lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -187,7 +187,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Toiler is a super efficient AWS SQS thread based message processor

data/celluloid-task-pooledfiber/.gitignore

@@ -1,9 +0,0 @@
-/.bundle/
-/.yardoc
-/Gemfile.lock
-/_yardoc/
-/coverage/
-/doc/
-/pkg/
-/spec/reports/
-/tmp/

data/celluloid-task-pooledfiber/.rspec

@@ -1,2 +0,0 @@
---format documentation
---color

data/celluloid-task-pooledfiber/.travis.yml

@@ -1,5 +0,0 @@
-language: ruby
-rvm:
-  - 2.2.1
-  - rbx-2
-  - jruby

data/celluloid-task-pooledfiber/Gemfile

@@ -1,11 +0,0 @@
-source 'https://rubygems.org'
-
-gem 'celluloid', github: 'celluloid/celluloid', branch: '0.17.0-prerelease'
-
-gemspec
-
-group :test do
-  gem 'rake'
-  gem 'rspec', '~> 3.2'
-  gem 'rspec-retry'
-end

data/celluloid-task-pooledfiber/LICENSE.txt

@@ -1,21 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2015 Chris Heald
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.

data/celluloid-task-pooledfiber/README.md

@@ -1,37 +0,0 @@
-# Celluloid::Task::PooledFiber
-
-[![Build Status](https://travis-ci.org/celluloid/celluloid-task-pooledfiber.svg)](https://travis-ci.org/celluloid/celluloid-task-pooledfiber)
-
-Provides a Task implementation which reuses a pool of fibers to run tasks. This improves Celluloid's performance.
-
-## Installation
-
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'celluloid-task-pooledfiber'
-```
-
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install celluloid-task-pooledfiber
-
-## Usage
-
-In your application, before starting Celluloid:
-
-    Celluloid.task_class = Celluloid::Task::PooledFiber
-
-That's it.
-
-## Contributing
-
-1. Fork it ( https://github.com/cheald/celluloid-task-pooledfiber/fork )
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create a new Pull Request