toiler 0.3.6 → 0.4.0.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
 - data/.gitignore +35 -35
 - data/.gitmodules +4 -4
 - data/.ruby-version +1 -1
 - data/Gemfile +9 -9
 - data/Gemfile.lock +41 -41
 - data/LICENSE +6 -6
 - data/README.md +125 -125
 - data/Rakefile +1 -1
 - data/bin/toiler +12 -12
 - data/lib/toiler.rb +55 -55
 - data/lib/toiler/actor/fetcher.rb +116 -116
 - data/lib/toiler/actor/processor.rb +124 -123
 - data/lib/toiler/actor/supervisor.rb +55 -55
 - data/lib/toiler/actor/utils/actor_logging.rb +28 -28
 - data/lib/toiler/aws/message.rb +64 -64
 - data/lib/toiler/aws/queue.rb +61 -61
 - data/lib/toiler/cli.rb +164 -164
 - data/lib/toiler/utils/argument_parser.rb +50 -50
 - data/lib/toiler/utils/environment_loader.rb +104 -104
 - data/lib/toiler/utils/logging.rb +41 -41
 - data/lib/toiler/version.rb +4 -4
 - data/lib/toiler/worker.rb +62 -62
 - data/toiler.gemspec +30 -30
 - metadata +4 -4
 
| 
         @@ -1,50 +1,50 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            module Toiler
         
     | 
| 
       2 
     | 
    
         
            -
              module Utils
         
     | 
| 
       3 
     | 
    
         
            -
                # Parses command-line arguments
         
     | 
| 
       4 
     | 
    
         
            -
                module ArgumentParser
         
     | 
| 
       5 
     | 
    
         
            -
                  module_function
         
     | 
| 
       6 
     | 
    
         
            -
             
     | 
| 
       7 
     | 
    
         
            -
                  def parse(argv)
         
     | 
| 
       8 
     | 
    
         
            -
                    opts = { queues: [] }
         
     | 
| 
       9 
     | 
    
         
            -
             
     | 
| 
       10 
     | 
    
         
            -
                    parser = OptionParser.new do |o|
         
     | 
| 
       11 
     | 
    
         
            -
                      o.on '-d', '--daemon', 'Daemonize process' do |arg|
         
     | 
| 
       12 
     | 
    
         
            -
                        opts[:daemon] = arg
         
     | 
| 
       13 
     | 
    
         
            -
                      end
         
     | 
| 
       14 
     | 
    
         
            -
             
     | 
| 
       15 
     | 
    
         
            -
                      o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
         
     | 
| 
       16 
     | 
    
         
            -
                        opts[:require] = arg
         
     | 
| 
       17 
     | 
    
         
            -
                      end
         
     | 
| 
       18 
     | 
    
         
            -
             
     | 
| 
       19 
     | 
    
         
            -
                      o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
         
     | 
| 
       20 
     | 
    
         
            -
                        opts[:config_file] = arg
         
     | 
| 
       21 
     | 
    
         
            -
                      end
         
     | 
| 
       22 
     | 
    
         
            -
             
     | 
| 
       23 
     | 
    
         
            -
                      o.on '-R', '--rails', 'Load Rails' do |arg|
         
     | 
| 
       24 
     | 
    
         
            -
                        opts[:rails] = arg
         
     | 
| 
       25 
     | 
    
         
            -
                      end
         
     | 
| 
       26 
     | 
    
         
            -
             
     | 
| 
       27 
     | 
    
         
            -
                      o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
         
     | 
| 
       28 
     | 
    
         
            -
                        opts[:logfile] = arg
         
     | 
| 
       29 
     | 
    
         
            -
                      end
         
     | 
| 
       30 
     | 
    
         
            -
             
     | 
| 
       31 
     | 
    
         
            -
                      o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
         
     | 
| 
       32 
     | 
    
         
            -
                        opts[:pidfile] = arg
         
     | 
| 
       33 
     | 
    
         
            -
                      end
         
     | 
| 
       34 
     | 
    
         
            -
             
     | 
| 
       35 
     | 
    
         
            -
                      o.on '-v', '--verbose', 'Print more verbose output' do |arg|
         
     | 
| 
       36 
     | 
    
         
            -
                        opts[:verbose] = arg
         
     | 
| 
       37 
     | 
    
         
            -
                      end
         
     | 
| 
       38 
     | 
    
         
            -
                    end
         
     | 
| 
       39 
     | 
    
         
            -
             
     | 
| 
       40 
     | 
    
         
            -
                    parser.banner = 'toiler [options]'
         
     | 
| 
       41 
     | 
    
         
            -
                    parser.on_tail '-h', '--help', 'Show help' do
         
     | 
| 
       42 
     | 
    
         
            -
                      puts parser
         
     | 
| 
       43 
     | 
    
         
            -
                      exit 1
         
     | 
| 
       44 
     | 
    
         
            -
                    end
         
     | 
| 
       45 
     | 
    
         
            -
                    parser.parse!(argv)
         
     | 
| 
       46 
     | 
    
         
            -
                    opts
         
     | 
| 
       47 
     | 
    
         
            -
                  end
         
     | 
| 
       48 
     | 
    
         
            -
                end
         
     | 
| 
       49 
     | 
    
         
            -
              end
         
     | 
| 
       50 
     | 
    
         
            -
            end
         
     | 
| 
      
 1 
     | 
    
         
            +
            module Toiler
         
     | 
| 
      
 2 
     | 
    
         
            +
              module Utils
         
     | 
| 
      
 3 
     | 
    
         
            +
                # Parses command-line arguments
         
     | 
| 
      
 4 
     | 
    
         
            +
                module ArgumentParser
         
     | 
| 
      
 5 
     | 
    
         
            +
                  module_function
         
     | 
| 
      
 6 
     | 
    
         
            +
             
     | 
| 
      
 7 
     | 
    
         
            +
                  def parse(argv)
         
     | 
| 
      
 8 
     | 
    
         
            +
                    opts = { queues: [] }
         
     | 
| 
      
 9 
     | 
    
         
            +
             
     | 
| 
      
 10 
     | 
    
         
            +
                    parser = OptionParser.new do |o|
         
     | 
| 
      
 11 
     | 
    
         
            +
                      o.on '-d', '--daemon', 'Daemonize process' do |arg|
         
     | 
| 
      
 12 
     | 
    
         
            +
                        opts[:daemon] = arg
         
     | 
| 
      
 13 
     | 
    
         
            +
                      end
         
     | 
| 
      
 14 
     | 
    
         
            +
             
     | 
| 
      
 15 
     | 
    
         
            +
                      o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
         
     | 
| 
      
 16 
     | 
    
         
            +
                        opts[:require] = arg
         
     | 
| 
      
 17 
     | 
    
         
            +
                      end
         
     | 
| 
      
 18 
     | 
    
         
            +
             
     | 
| 
      
 19 
     | 
    
         
            +
                      o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
         
     | 
| 
      
 20 
     | 
    
         
            +
                        opts[:config_file] = arg
         
     | 
| 
      
 21 
     | 
    
         
            +
                      end
         
     | 
| 
      
 22 
     | 
    
         
            +
             
     | 
| 
      
 23 
     | 
    
         
            +
                      o.on '-R', '--rails', 'Load Rails' do |arg|
         
     | 
| 
      
 24 
     | 
    
         
            +
                        opts[:rails] = arg
         
     | 
| 
      
 25 
     | 
    
         
            +
                      end
         
     | 
| 
      
 26 
     | 
    
         
            +
             
     | 
| 
      
 27 
     | 
    
         
            +
                      o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
         
     | 
| 
      
 28 
     | 
    
         
            +
                        opts[:logfile] = arg
         
     | 
| 
      
 29 
     | 
    
         
            +
                      end
         
     | 
| 
      
 30 
     | 
    
         
            +
             
     | 
| 
      
 31 
     | 
    
         
            +
                      o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
         
     | 
| 
      
 32 
     | 
    
         
            +
                        opts[:pidfile] = arg
         
     | 
| 
      
 33 
     | 
    
         
            +
                      end
         
     | 
| 
      
 34 
     | 
    
         
            +
             
     | 
| 
      
 35 
     | 
    
         
            +
                      o.on '-v', '--verbose', 'Print more verbose output' do |arg|
         
     | 
| 
      
 36 
     | 
    
         
            +
                        opts[:verbose] = arg
         
     | 
| 
      
 37 
     | 
    
         
            +
                      end
         
     | 
| 
      
 38 
     | 
    
         
            +
                    end
         
     | 
| 
      
 39 
     | 
    
         
            +
             
     | 
| 
      
 40 
     | 
    
         
            +
                    parser.banner = 'toiler [options]'
         
     | 
| 
      
 41 
     | 
    
         
            +
                    parser.on_tail '-h', '--help', 'Show help' do
         
     | 
| 
      
 42 
     | 
    
         
            +
                      puts parser
         
     | 
| 
      
 43 
     | 
    
         
            +
                      exit 1
         
     | 
| 
      
 44 
     | 
    
         
            +
                    end
         
     | 
| 
      
 45 
     | 
    
         
            +
                    parser.parse!(argv)
         
     | 
| 
      
 46 
     | 
    
         
            +
                    opts
         
     | 
| 
      
 47 
     | 
    
         
            +
                  end
         
     | 
| 
      
 48 
     | 
    
         
            +
                end
         
     | 
| 
      
 49 
     | 
    
         
            +
              end
         
     | 
| 
      
 50 
     | 
    
         
            +
            end
         
     | 
| 
         @@ -1,104 +1,104 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            require 'erb'
         
     | 
| 
       2 
     | 
    
         
            -
            require 'yaml'
         
     | 
| 
       3 
     | 
    
         
            -
             
     | 
| 
       4 
     | 
    
         
            -
            module Toiler
         
     | 
| 
       5 
     | 
    
         
            -
              module Utils
         
     | 
| 
       6 
     | 
    
         
            -
                # Takes care of loading componentes to get toiler ready to run
         
     | 
| 
       7 
     | 
    
         
            -
                class EnvironmentLoader
         
     | 
| 
       8 
     | 
    
         
            -
                  attr_reader :options
         
     | 
| 
       9 
     | 
    
         
            -
             
     | 
| 
       10 
     | 
    
         
            -
                  def self.load(options)
         
     | 
| 
       11 
     | 
    
         
            -
                    new(options).load
         
     | 
| 
       12 
     | 
    
         
            -
                  end
         
     | 
| 
       13 
     | 
    
         
            -
             
     | 
| 
       14 
     | 
    
         
            -
                  def self.load_for_rails_console
         
     | 
| 
       15 
     | 
    
         
            -
                    load(config_file: (Rails.root + 'config' + 'toiler.yml'))
         
     | 
| 
       16 
     | 
    
         
            -
                  end
         
     | 
| 
       17 
     | 
    
         
            -
             
     | 
| 
       18 
     | 
    
         
            -
                  def initialize(options)
         
     | 
| 
       19 
     | 
    
         
            -
                    @options = options
         
     | 
| 
       20 
     | 
    
         
            -
                  end
         
     | 
| 
       21 
     | 
    
         
            -
             
     | 
| 
       22 
     | 
    
         
            -
                  def load
         
     | 
| 
       23 
     | 
    
         
            -
                    initialize_logger
         
     | 
| 
       24 
     | 
    
         
            -
                    load_rails if options[:rails]
         
     | 
| 
       25 
     | 
    
         
            -
                    require_workers if options[:require]
         
     | 
| 
       26 
     | 
    
         
            -
                    Toiler.options.merge!(config_file_options)
         
     | 
| 
       27 
     | 
    
         
            -
                    Toiler.options.merge!(options)
         
     | 
| 
       28 
     | 
    
         
            -
                    initialize_aws
         
     | 
| 
       29 
     | 
    
         
            -
                  end
         
     | 
| 
       30 
     | 
    
         
            -
             
     | 
| 
       31 
     | 
    
         
            -
                  private
         
     | 
| 
       32 
     | 
    
         
            -
             
     | 
| 
       33 
     | 
    
         
            -
                  def config_file_options
         
     | 
| 
       34 
     | 
    
         
            -
                    if (path = options[:config_file])
         
     | 
| 
       35 
     | 
    
         
            -
                      unless File.exist?(path)
         
     | 
| 
       36 
     | 
    
         
            -
                        Toiler.logger.warn "Config file #{path} does not exist"
         
     | 
| 
       37 
     | 
    
         
            -
                        path = nil
         
     | 
| 
       38 
     | 
    
         
            -
                      end
         
     | 
| 
       39 
     | 
    
         
            -
                    end
         
     | 
| 
       40 
     | 
    
         
            -
             
     | 
| 
       41 
     | 
    
         
            -
                    return {} unless path
         
     | 
| 
       42 
     | 
    
         
            -
             
     | 
| 
       43 
     | 
    
         
            -
                    deep_symbolize_keys YAML.load(ERB.new(File.read(path)).result)
         
     | 
| 
       44 
     | 
    
         
            -
                  end
         
     | 
| 
       45 
     | 
    
         
            -
             
     | 
| 
       46 
     | 
    
         
            -
                  def initialize_aws
         
     | 
| 
       47 
     | 
    
         
            -
                    return if Toiler.options[:aws].empty?
         
     | 
| 
       48 
     | 
    
         
            -
                    ::Aws.config[:region] = Toiler.options[:aws][:region]
         
     | 
| 
       49 
     | 
    
         
            -
                    set_aws_credentials
         
     | 
| 
       50 
     | 
    
         
            -
                  end
         
     | 
| 
       51 
     | 
    
         
            -
             
     | 
| 
       52 
     | 
    
         
            -
                  def set_aws_credentials
         
     | 
| 
       53 
     | 
    
         
            -
                    return unless Toiler.options[:aws][:access_key_id]
         
     | 
| 
       54 
     | 
    
         
            -
                    ::Aws.config[:credentials] = ::Aws::Credentials.new(
         
     | 
| 
       55 
     | 
    
         
            -
                      Toiler.options[:aws][:access_key_id],
         
     | 
| 
       56 
     | 
    
         
            -
                      Toiler.options[:aws][:secret_access_key]
         
     | 
| 
       57 
     | 
    
         
            -
                    )
         
     | 
| 
       58 
     | 
    
         
            -
                  end
         
     | 
| 
       59 
     | 
    
         
            -
             
     | 
| 
       60 
     | 
    
         
            -
                  def initialize_logger
         
     | 
| 
       61 
     | 
    
         
            -
                    Toiler::Utils::Logging.initialize_logger(options[:logfile])
         
     | 
| 
       62 
     | 
    
         
            -
                    Toiler.logger.level = Logger::DEBUG if options[:verbose]
         
     | 
| 
       63 
     | 
    
         
            -
                  end
         
     | 
| 
       64 
     | 
    
         
            -
             
     | 
| 
       65 
     | 
    
         
            -
                  def load_rails
         
     | 
| 
       66 
     | 
    
         
            -
                    require 'rails'
         
     | 
| 
       67 
     | 
    
         
            -
                    if ::Rails::VERSION::MAJOR < 4
         
     | 
| 
       68 
     | 
    
         
            -
                      load_rails_old
         
     | 
| 
       69 
     | 
    
         
            -
                    else
         
     | 
| 
       70 
     | 
    
         
            -
                      load_rails_new
         
     | 
| 
       71 
     | 
    
         
            -
                    end
         
     | 
| 
       72 
     | 
    
         
            -
                    Toiler.logger.info 'Rails environment loaded'
         
     | 
| 
       73 
     | 
    
         
            -
                  end
         
     | 
| 
       74 
     | 
    
         
            -
             
     | 
| 
       75 
     | 
    
         
            -
                  def load_rails_old
         
     | 
| 
       76 
     | 
    
         
            -
                    require File.expand_path('config/environment.rb')
         
     | 
| 
       77 
     | 
    
         
            -
                    ::Rails.application.eager_load!
         
     | 
| 
       78 
     | 
    
         
            -
                  end
         
     | 
| 
       79 
     | 
    
         
            -
             
     | 
| 
       80 
     | 
    
         
            -
                  def load_rails_new
         
     | 
| 
       81 
     | 
    
         
            -
                    require File.expand_path('config/application.rb')
         
     | 
| 
       82 
     | 
    
         
            -
                    ::Rails::Application.initializer 'toiler.eager_load' do
         
     | 
| 
       83 
     | 
    
         
            -
                      ::Rails.application.config.eager_load = true
         
     | 
| 
       84 
     | 
    
         
            -
                    end
         
     | 
| 
       85 
     | 
    
         
            -
                    require File.expand_path('config/environment.rb')
         
     | 
| 
       86 
     | 
    
         
            -
                  end
         
     | 
| 
       87 
     | 
    
         
            -
             
     | 
| 
       88 
     | 
    
         
            -
                  def require_workers
         
     | 
| 
       89 
     | 
    
         
            -
                    require options[:require]
         
     | 
| 
       90 
     | 
    
         
            -
                  end
         
     | 
| 
       91 
     | 
    
         
            -
             
     | 
| 
       92 
     | 
    
         
            -
                  def deep_symbolize_keys(h)
         
     | 
| 
       93 
     | 
    
         
            -
                    h.each_with_object({}) do |(key, value), result|
         
     | 
| 
       94 
     | 
    
         
            -
                      k = key.respond_to?(:to_sym) ? key.to_sym : key
         
     | 
| 
       95 
     | 
    
         
            -
                      result[k] = if value.is_a? Hash
         
     | 
| 
       96 
     | 
    
         
            -
                                    deep_symbolize_keys value
         
     | 
| 
       97 
     | 
    
         
            -
                                  else
         
     | 
| 
       98 
     | 
    
         
            -
                                    value
         
     | 
| 
       99 
     | 
    
         
            -
                                  end
         
     | 
| 
       100 
     | 
    
         
            -
                    end
         
     | 
| 
       101 
     | 
    
         
            -
                  end
         
     | 
| 
       102 
     | 
    
         
            -
                end
         
     | 
| 
       103 
     | 
    
         
            -
              end
         
     | 
| 
       104 
     | 
    
         
            -
            end
         
     | 
| 
      
 1 
     | 
    
         
            +
            require 'erb'
         
     | 
| 
      
 2 
     | 
    
         
            +
            require 'yaml'
         
     | 
| 
      
 3 
     | 
    
         
            +
             
     | 
| 
      
 4 
     | 
    
         
            +
            module Toiler
         
     | 
| 
      
 5 
     | 
    
         
            +
              module Utils
         
     | 
| 
      
 6 
     | 
    
         
            +
                # Takes care of loading componentes to get toiler ready to run
         
     | 
| 
      
 7 
     | 
    
         
            +
                class EnvironmentLoader
         
     | 
| 
      
 8 
     | 
    
         
            +
                  attr_reader :options
         
     | 
| 
      
 9 
     | 
    
         
            +
             
     | 
| 
      
 10 
     | 
    
         
            +
                  def self.load(options)
         
     | 
| 
      
 11 
     | 
    
         
            +
                    new(options).load
         
     | 
| 
      
 12 
     | 
    
         
            +
                  end
         
     | 
| 
      
 13 
     | 
    
         
            +
             
     | 
| 
      
 14 
     | 
    
         
            +
                  def self.load_for_rails_console
         
     | 
| 
      
 15 
     | 
    
         
            +
                    load(config_file: (Rails.root + 'config' + 'toiler.yml'))
         
     | 
| 
      
 16 
     | 
    
         
            +
                  end
         
     | 
| 
      
 17 
     | 
    
         
            +
             
     | 
| 
      
 18 
     | 
    
         
            +
                  def initialize(options)
         
     | 
| 
      
 19 
     | 
    
         
            +
                    @options = options
         
     | 
| 
      
 20 
     | 
    
         
            +
                  end
         
     | 
| 
      
 21 
     | 
    
         
            +
             
     | 
| 
      
 22 
     | 
    
         
            +
                  def load
         
     | 
| 
      
 23 
     | 
    
         
            +
                    initialize_logger
         
     | 
| 
      
 24 
     | 
    
         
            +
                    load_rails if options[:rails]
         
     | 
| 
      
 25 
     | 
    
         
            +
                    require_workers if options[:require]
         
     | 
| 
      
 26 
     | 
    
         
            +
                    Toiler.options.merge!(config_file_options)
         
     | 
| 
      
 27 
     | 
    
         
            +
                    Toiler.options.merge!(options)
         
     | 
| 
      
 28 
     | 
    
         
            +
                    initialize_aws
         
     | 
| 
      
 29 
     | 
    
         
            +
                  end
         
     | 
| 
      
 30 
     | 
    
         
            +
             
     | 
| 
      
 31 
     | 
    
         
            +
                  private
         
     | 
| 
      
 32 
     | 
    
         
            +
             
     | 
| 
      
 33 
     | 
    
         
            +
                  def config_file_options
         
     | 
| 
      
 34 
     | 
    
         
            +
                    if (path = options[:config_file])
         
     | 
| 
      
 35 
     | 
    
         
            +
                      unless File.exist?(path)
         
     | 
| 
      
 36 
     | 
    
         
            +
                        Toiler.logger.warn "Config file #{path} does not exist"
         
     | 
| 
      
 37 
     | 
    
         
            +
                        path = nil
         
     | 
| 
      
 38 
     | 
    
         
            +
                      end
         
     | 
| 
      
 39 
     | 
    
         
            +
                    end
         
     | 
| 
      
 40 
     | 
    
         
            +
             
     | 
| 
      
 41 
     | 
    
         
            +
                    return {} unless path
         
     | 
| 
      
 42 
     | 
    
         
            +
             
     | 
| 
      
 43 
     | 
    
         
            +
                    deep_symbolize_keys YAML.load(ERB.new(File.read(path)).result)
         
     | 
| 
      
 44 
     | 
    
         
            +
                  end
         
     | 
| 
      
 45 
     | 
    
         
            +
             
     | 
| 
      
 46 
     | 
    
         
            +
                  def initialize_aws
         
     | 
| 
      
 47 
     | 
    
         
            +
                    return if Toiler.options[:aws].empty?
         
     | 
| 
      
 48 
     | 
    
         
            +
                    ::Aws.config[:region] = Toiler.options[:aws][:region]
         
     | 
| 
      
 49 
     | 
    
         
            +
                    set_aws_credentials
         
     | 
| 
      
 50 
     | 
    
         
            +
                  end
         
     | 
| 
      
 51 
     | 
    
         
            +
             
     | 
| 
      
 52 
     | 
    
         
            +
                  def set_aws_credentials
         
     | 
| 
      
 53 
     | 
    
         
            +
                    return unless Toiler.options[:aws][:access_key_id]
         
     | 
| 
      
 54 
     | 
    
         
            +
                    ::Aws.config[:credentials] = ::Aws::Credentials.new(
         
     | 
| 
      
 55 
     | 
    
         
            +
                      Toiler.options[:aws][:access_key_id],
         
     | 
| 
      
 56 
     | 
    
         
            +
                      Toiler.options[:aws][:secret_access_key]
         
     | 
| 
      
 57 
     | 
    
         
            +
                    )
         
     | 
| 
      
 58 
     | 
    
         
            +
                  end
         
     | 
| 
      
 59 
     | 
    
         
            +
             
     | 
| 
      
 60 
     | 
    
         
            +
                  def initialize_logger
         
     | 
| 
      
 61 
     | 
    
         
            +
                    Toiler::Utils::Logging.initialize_logger(options[:logfile])
         
     | 
| 
      
 62 
     | 
    
         
            +
                    Toiler.logger.level = Logger::DEBUG if options[:verbose]
         
     | 
| 
      
 63 
     | 
    
         
            +
                  end
         
     | 
| 
      
 64 
     | 
    
         
            +
             
     | 
| 
      
 65 
     | 
    
         
            +
                  def load_rails
         
     | 
| 
      
 66 
     | 
    
         
            +
                    require 'rails'
         
     | 
| 
      
 67 
     | 
    
         
            +
                    if ::Rails::VERSION::MAJOR < 4
         
     | 
| 
      
 68 
     | 
    
         
            +
                      load_rails_old
         
     | 
| 
      
 69 
     | 
    
         
            +
                    else
         
     | 
| 
      
 70 
     | 
    
         
            +
                      load_rails_new
         
     | 
| 
      
 71 
     | 
    
         
            +
                    end
         
     | 
| 
      
 72 
     | 
    
         
            +
                    Toiler.logger.info 'Rails environment loaded'
         
     | 
| 
      
 73 
     | 
    
         
            +
                  end
         
     | 
| 
      
 74 
     | 
    
         
            +
             
     | 
| 
      
 75 
     | 
    
         
            +
                  def load_rails_old
         
     | 
| 
      
 76 
     | 
    
         
            +
                    require File.expand_path('config/environment.rb')
         
     | 
| 
      
 77 
     | 
    
         
            +
                    ::Rails.application.eager_load!
         
     | 
| 
      
 78 
     | 
    
         
            +
                  end
         
     | 
| 
      
 79 
     | 
    
         
            +
             
     | 
| 
      
 80 
     | 
    
         
            +
                  def load_rails_new
         
     | 
| 
      
 81 
     | 
    
         
            +
                    require File.expand_path('config/application.rb')
         
     | 
| 
      
 82 
     | 
    
         
            +
                    ::Rails::Application.initializer 'toiler.eager_load' do
         
     | 
| 
      
 83 
     | 
    
         
            +
                      ::Rails.application.config.eager_load = true
         
     | 
| 
      
 84 
     | 
    
         
            +
                    end
         
     | 
| 
      
 85 
     | 
    
         
            +
                    require File.expand_path('config/environment.rb')
         
     | 
| 
      
 86 
     | 
    
         
            +
                  end
         
     | 
| 
      
 87 
     | 
    
         
            +
             
     | 
| 
      
 88 
     | 
    
         
            +
                  def require_workers
         
     | 
| 
      
 89 
     | 
    
         
            +
                    require options[:require]
         
     | 
| 
      
 90 
     | 
    
         
            +
                  end
         
     | 
| 
      
 91 
     | 
    
         
            +
             
     | 
| 
      
 92 
     | 
    
         
            +
                  def deep_symbolize_keys(h)
         
     | 
| 
      
 93 
     | 
    
         
            +
                    h.each_with_object({}) do |(key, value), result|
         
     | 
| 
      
 94 
     | 
    
         
            +
                      k = key.respond_to?(:to_sym) ? key.to_sym : key
         
     | 
| 
      
 95 
     | 
    
         
            +
                      result[k] = if value.is_a? Hash
         
     | 
| 
      
 96 
     | 
    
         
            +
                                    deep_symbolize_keys value
         
     | 
| 
      
 97 
     | 
    
         
            +
                                  else
         
     | 
| 
      
 98 
     | 
    
         
            +
                                    value
         
     | 
| 
      
 99 
     | 
    
         
            +
                                  end
         
     | 
| 
      
 100 
     | 
    
         
            +
                    end
         
     | 
| 
      
 101 
     | 
    
         
            +
                  end
         
     | 
| 
      
 102 
     | 
    
         
            +
                end
         
     | 
| 
      
 103 
     | 
    
         
            +
              end
         
     | 
| 
      
 104 
     | 
    
         
            +
            end
         
     | 
    
        data/lib/toiler/utils/logging.rb
    CHANGED
    
    | 
         @@ -1,41 +1,41 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            require 'time'
         
     | 
| 
       2 
     | 
    
         
            -
            require 'logger'
         
     | 
| 
       3 
     | 
    
         
            -
             
     | 
| 
       4 
     | 
    
         
            -
            module Toiler
         
     | 
| 
       5 
     | 
    
         
            -
              module Utils
         
     | 
| 
       6 
     | 
    
         
            -
                # Initializes and exposes Toiler's default logger
         
     | 
| 
       7 
     | 
    
         
            -
                module Logging
         
     | 
| 
       8 
     | 
    
         
            -
                  # Toiler's default log formatter
         
     | 
| 
       9 
     | 
    
         
            -
                  class Pretty < Logger::Formatter
         
     | 
| 
       10 
     | 
    
         
            -
                    def call(sev, time, progname, msg)
         
     | 
| 
       11 
     | 
    
         
            -
                      formatted = msg.respond_to?(:gsub) ? msg.gsub("\n", "\n\t") : msg
         
     | 
| 
       12 
     | 
    
         
            -
                      time = time.utc.iso8601
         
     | 
| 
       13 
     | 
    
         
            -
                      pid = Process.pid
         
     | 
| 
       14 
     | 
    
         
            -
                      if progname.to_s.empty?
         
     | 
| 
       15 
     | 
    
         
            -
                        "#{time} Pid:#{pid} Level:#{sev}: #{formatted}\n"
         
     | 
| 
       16 
     | 
    
         
            -
                      else
         
     | 
| 
       17 
     | 
    
         
            -
                        "#{time} Pid:#{pid} Actor:#{progname} Level:#{sev}: #{formatted}\n"
         
     | 
| 
       18 
     | 
    
         
            -
                      end
         
     | 
| 
       19 
     | 
    
         
            -
                    end
         
     | 
| 
       20 
     | 
    
         
            -
                  end
         
     | 
| 
       21 
     | 
    
         
            -
             
     | 
| 
       22 
     | 
    
         
            -
                  module_function
         
     | 
| 
       23 
     | 
    
         
            -
             
     | 
| 
       24 
     | 
    
         
            -
                  def initialize_logger(log_target = STDOUT)
         
     | 
| 
       25 
     | 
    
         
            -
                    log_target = STDOUT if log_target.nil?
         
     | 
| 
       26 
     | 
    
         
            -
                    @logger = Logger.new(log_target)
         
     | 
| 
       27 
     | 
    
         
            -
                    @logger.level = Logger::INFO
         
     | 
| 
       28 
     | 
    
         
            -
                    @logger.formatter = Pretty.new
         
     | 
| 
       29 
     | 
    
         
            -
                    @logger
         
     | 
| 
       30 
     | 
    
         
            -
                  end
         
     | 
| 
       31 
     | 
    
         
            -
             
     | 
| 
       32 
     | 
    
         
            -
                  def logger
         
     | 
| 
       33 
     | 
    
         
            -
                    @logger || initialize_logger
         
     | 
| 
       34 
     | 
    
         
            -
                  end
         
     | 
| 
       35 
     | 
    
         
            -
             
     | 
| 
       36 
     | 
    
         
            -
                  def logger=(log)
         
     | 
| 
       37 
     | 
    
         
            -
                    @logger = (log ? log : Logger.new('/dev/null'))
         
     | 
| 
       38 
     | 
    
         
            -
                  end
         
     | 
| 
       39 
     | 
    
         
            -
                end
         
     | 
| 
       40 
     | 
    
         
            -
              end
         
     | 
| 
       41 
     | 
    
         
            -
            end
         
     | 
| 
      
 1 
     | 
    
         
            +
            require 'time'
         
     | 
| 
      
 2 
     | 
    
         
            +
            require 'logger'
         
     | 
| 
      
 3 
     | 
    
         
            +
             
     | 
| 
      
 4 
     | 
    
         
            +
            module Toiler
         
     | 
| 
      
 5 
     | 
    
         
            +
              module Utils
         
     | 
| 
      
 6 
     | 
    
         
            +
                # Initializes and exposes Toiler's default logger
         
     | 
| 
      
 7 
     | 
    
         
            +
                module Logging
         
     | 
| 
      
 8 
     | 
    
         
            +
                  # Toiler's default log formatter
         
     | 
| 
      
 9 
     | 
    
         
            +
                  class Pretty < Logger::Formatter
         
     | 
| 
      
 10 
     | 
    
         
            +
                    def call(sev, time, progname, msg)
         
     | 
| 
      
 11 
     | 
    
         
            +
                      formatted = msg.respond_to?(:gsub) ? msg.gsub("\n", "\n\t") : msg
         
     | 
| 
      
 12 
     | 
    
         
            +
                      time = time.utc.iso8601
         
     | 
| 
      
 13 
     | 
    
         
            +
                      pid = Process.pid
         
     | 
| 
      
 14 
     | 
    
         
            +
                      if progname.to_s.empty?
         
     | 
| 
      
 15 
     | 
    
         
            +
                        "#{time} Pid:#{pid} Level:#{sev}: #{formatted}\n"
         
     | 
| 
      
 16 
     | 
    
         
            +
                      else
         
     | 
| 
      
 17 
     | 
    
         
            +
                        "#{time} Pid:#{pid} Actor:#{progname} Level:#{sev}: #{formatted}\n"
         
     | 
| 
      
 18 
     | 
    
         
            +
                      end
         
     | 
| 
      
 19 
     | 
    
         
            +
                    end
         
     | 
| 
      
 20 
     | 
    
         
            +
                  end
         
     | 
| 
      
 21 
     | 
    
         
            +
             
     | 
| 
      
 22 
     | 
    
         
            +
                  module_function
         
     | 
| 
      
 23 
     | 
    
         
            +
             
     | 
| 
      
 24 
     | 
    
         
            +
                  def initialize_logger(log_target = STDOUT)
         
     | 
| 
      
 25 
     | 
    
         
            +
                    log_target = STDOUT if log_target.nil?
         
     | 
| 
      
 26 
     | 
    
         
            +
                    @logger = Logger.new(log_target)
         
     | 
| 
      
 27 
     | 
    
         
            +
                    @logger.level = Logger::INFO
         
     | 
| 
      
 28 
     | 
    
         
            +
                    @logger.formatter = Pretty.new
         
     | 
| 
      
 29 
     | 
    
         
            +
                    @logger
         
     | 
| 
      
 30 
     | 
    
         
            +
                  end
         
     | 
| 
      
 31 
     | 
    
         
            +
             
     | 
| 
      
 32 
     | 
    
         
            +
                  def logger
         
     | 
| 
      
 33 
     | 
    
         
            +
                    @logger || initialize_logger
         
     | 
| 
      
 34 
     | 
    
         
            +
                  end
         
     | 
| 
      
 35 
     | 
    
         
            +
             
     | 
| 
      
 36 
     | 
    
         
            +
                  def logger=(log)
         
     | 
| 
      
 37 
     | 
    
         
            +
                    @logger = (log ? log : Logger.new('/dev/null'))
         
     | 
| 
      
 38 
     | 
    
         
            +
                  end
         
     | 
| 
      
 39 
     | 
    
         
            +
                end
         
     | 
| 
      
 40 
     | 
    
         
            +
              end
         
     | 
| 
      
 41 
     | 
    
         
            +
            end
         
     | 
    
        data/lib/toiler/version.rb
    CHANGED
    
    | 
         @@ -1,4 +1,4 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            # Toiler Version
         
     | 
| 
       2 
     | 
    
         
            -
            module Toiler
         
     | 
| 
       3 
     | 
    
         
            -
              VERSION = '0. 
     | 
| 
       4 
     | 
    
         
            -
            end
         
     | 
| 
      
 1 
     | 
    
         
            +
            # Toiler Version
         
     | 
| 
      
 2 
     | 
    
         
            +
            module Toiler
         
     | 
| 
      
 3 
     | 
    
         
            +
              VERSION = '0.4.0.beta1'.freeze
         
     | 
| 
      
 4 
     | 
    
         
            +
            end
         
     | 
    
        data/lib/toiler/worker.rb
    CHANGED
    
    | 
         @@ -1,62 +1,62 @@ 
     | 
|
| 
       1 
     | 
    
         
            -
            module Toiler
         
     | 
| 
       2 
     | 
    
         
            -
              # Toiler's Worker behaviour
         
     | 
| 
       3 
     | 
    
         
            -
              module Worker
         
     | 
| 
       4 
     | 
    
         
            -
                def self.included(base)
         
     | 
| 
       5 
     | 
    
         
            -
                  base.extend(ClassMethods)
         
     | 
| 
       6 
     | 
    
         
            -
                  base.class_variable_set(:@@toiler_options, Toiler.default_options)
         
     | 
| 
       7 
     | 
    
         
            -
                end
         
     | 
| 
       8 
     | 
    
         
            -
             
     | 
| 
       9 
     | 
    
         
            -
                def log(level, message)
         
     | 
| 
       10 
     | 
    
         
            -
                  Toiler.logger.log(level, message, self.class)
         
     | 
| 
       11 
     | 
    
         
            -
                end
         
     | 
| 
       12 
     | 
    
         
            -
             
     | 
| 
       13 
     | 
    
         
            -
                def error(msg)
         
     | 
| 
       14 
     | 
    
         
            -
                  log Logger::Severity::ERROR, msg
         
     | 
| 
       15 
     | 
    
         
            -
                end
         
     | 
| 
       16 
     | 
    
         
            -
             
     | 
| 
       17 
     | 
    
         
            -
                def info(msg)
         
     | 
| 
       18 
     | 
    
         
            -
                  log Logger::Severity::INFO, msg
         
     | 
| 
       19 
     | 
    
         
            -
                end
         
     | 
| 
       20 
     | 
    
         
            -
             
     | 
| 
       21 
     | 
    
         
            -
                def debug(msg)
         
     | 
| 
       22 
     | 
    
         
            -
                  log Logger::Severity::DEBUG, msg
         
     | 
| 
       23 
     | 
    
         
            -
                end
         
     | 
| 
       24 
     | 
    
         
            -
             
     | 
| 
       25 
     | 
    
         
            -
                def warn(msg)
         
     | 
| 
       26 
     | 
    
         
            -
                  log Logger::Severity::WARN, msg
         
     | 
| 
       27 
     | 
    
         
            -
                end
         
     | 
| 
       28 
     | 
    
         
            -
             
     | 
| 
       29 
     | 
    
         
            -
                def fatal(msg)
         
     | 
| 
       30 
     | 
    
         
            -
                  log Logger::Severity::FATAL, msg
         
     | 
| 
       31 
     | 
    
         
            -
                end
         
     | 
| 
       32 
     | 
    
         
            -
             
     | 
| 
       33 
     | 
    
         
            -
                # Class methods for Workers
         
     | 
| 
       34 
     | 
    
         
            -
                module ClassMethods
         
     | 
| 
       35 
     | 
    
         
            -
                  def toiler_options(options = {})
         
     | 
| 
       36 
     | 
    
         
            -
                    return class_variable_get(:@@toiler_options) if options.empty?
         
     | 
| 
       37 
     | 
    
         
            -
                    Toiler.worker_class_registry[options[:queue]] = self if options[:queue]
         
     | 
| 
       38 
     | 
    
         
            -
                    class_variable_get(:@@toiler_options).merge! options
         
     | 
| 
       39 
     | 
    
         
            -
                  end
         
     | 
| 
       40 
     | 
    
         
            -
             
     | 
| 
       41 
     | 
    
         
            -
                  def batch?
         
     | 
| 
       42 
     | 
    
         
            -
                    class_variable_get(:@@toiler_options)[:batch]
         
     | 
| 
       43 
     | 
    
         
            -
                  end
         
     | 
| 
       44 
     | 
    
         
            -
             
     | 
| 
       45 
     | 
    
         
            -
                  def concurrency
         
     | 
| 
       46 
     | 
    
         
            -
                    class_variable_get(:@@toiler_options)[:concurrency]
         
     | 
| 
       47 
     | 
    
         
            -
                  end
         
     | 
| 
       48 
     | 
    
         
            -
             
     | 
| 
       49 
     | 
    
         
            -
                  def queue
         
     | 
| 
       50 
     | 
    
         
            -
                    class_variable_get(:@@toiler_options)[:queue]
         
     | 
| 
       51 
     | 
    
         
            -
                  end
         
     | 
| 
       52 
     | 
    
         
            -
             
     | 
| 
       53 
     | 
    
         
            -
                  def auto_visibility_timeout?
         
     | 
| 
       54 
     | 
    
         
            -
                    class_variable_get(:@@toiler_options)[:auto_visibility_timeout]
         
     | 
| 
       55 
     | 
    
         
            -
                  end
         
     | 
| 
       56 
     | 
    
         
            -
             
     | 
| 
       57 
     | 
    
         
            -
                  def auto_delete?
         
     | 
| 
       58 
     | 
    
         
            -
                    class_variable_get(:@@toiler_options)[:auto_delete]
         
     | 
| 
       59 
     | 
    
         
            -
                  end
         
     | 
| 
       60 
     | 
    
         
            -
                end
         
     | 
| 
       61 
     | 
    
         
            -
              end
         
     | 
| 
       62 
     | 
    
         
            -
            end
         
     | 
| 
      
 1 
     | 
    
         
            +
            module Toiler
         
     | 
| 
      
 2 
     | 
    
         
            +
              # Toiler's Worker behaviour
         
     | 
| 
      
 3 
     | 
    
         
            +
              module Worker
         
     | 
| 
      
 4 
     | 
    
         
            +
                def self.included(base)
         
     | 
| 
      
 5 
     | 
    
         
            +
                  base.extend(ClassMethods)
         
     | 
| 
      
 6 
     | 
    
         
            +
                  base.class_variable_set(:@@toiler_options, Toiler.default_options)
         
     | 
| 
      
 7 
     | 
    
         
            +
                end
         
     | 
| 
      
 8 
     | 
    
         
            +
             
     | 
| 
      
 9 
     | 
    
         
            +
                def log(level, message)
         
     | 
| 
      
 10 
     | 
    
         
            +
                  Toiler.logger.log(level, message, self.class)
         
     | 
| 
      
 11 
     | 
    
         
            +
                end
         
     | 
| 
      
 12 
     | 
    
         
            +
             
     | 
| 
      
 13 
     | 
    
         
            +
                def error(msg)
         
     | 
| 
      
 14 
     | 
    
         
            +
                  log Logger::Severity::ERROR, msg
         
     | 
| 
      
 15 
     | 
    
         
            +
                end
         
     | 
| 
      
 16 
     | 
    
         
            +
             
     | 
| 
      
 17 
     | 
    
         
            +
                def info(msg)
         
     | 
| 
      
 18 
     | 
    
         
            +
                  log Logger::Severity::INFO, msg
         
     | 
| 
      
 19 
     | 
    
         
            +
                end
         
     | 
| 
      
 20 
     | 
    
         
            +
             
     | 
| 
      
 21 
     | 
    
         
            +
                def debug(msg)
         
     | 
| 
      
 22 
     | 
    
         
            +
                  log Logger::Severity::DEBUG, msg
         
     | 
| 
      
 23 
     | 
    
         
            +
                end
         
     | 
| 
      
 24 
     | 
    
         
            +
             
     | 
| 
      
 25 
     | 
    
         
            +
                def warn(msg)
         
     | 
| 
      
 26 
     | 
    
         
            +
                  log Logger::Severity::WARN, msg
         
     | 
| 
      
 27 
     | 
    
         
            +
                end
         
     | 
| 
      
 28 
     | 
    
         
            +
             
     | 
| 
      
 29 
     | 
    
         
            +
                def fatal(msg)
         
     | 
| 
      
 30 
     | 
    
         
            +
                  log Logger::Severity::FATAL, msg
         
     | 
| 
      
 31 
     | 
    
         
            +
                end
         
     | 
| 
      
 32 
     | 
    
         
            +
             
     | 
| 
      
 33 
     | 
    
         
            +
                # Class methods for Workers
         
     | 
| 
      
 34 
     | 
    
         
            +
                module ClassMethods
         
     | 
| 
      
 35 
     | 
    
         
            +
                  def toiler_options(options = {})
         
     | 
| 
      
 36 
     | 
    
         
            +
                    return class_variable_get(:@@toiler_options) if options.empty?
         
     | 
| 
      
 37 
     | 
    
         
            +
                    Toiler.worker_class_registry[options[:queue]] = self if options[:queue]
         
     | 
| 
      
 38 
     | 
    
         
            +
                    class_variable_get(:@@toiler_options).merge! options
         
     | 
| 
      
 39 
     | 
    
         
            +
                  end
         
     | 
| 
      
 40 
     | 
    
         
            +
             
     | 
| 
      
 41 
     | 
    
         
            +
                  def batch?
         
     | 
| 
      
 42 
     | 
    
         
            +
                    class_variable_get(:@@toiler_options)[:batch]
         
     | 
| 
      
 43 
     | 
    
         
            +
                  end
         
     | 
| 
      
 44 
     | 
    
         
            +
             
     | 
| 
      
 45 
     | 
    
         
            +
                  def concurrency
         
     | 
| 
      
 46 
     | 
    
         
            +
                    class_variable_get(:@@toiler_options)[:concurrency]
         
     | 
| 
      
 47 
     | 
    
         
            +
                  end
         
     | 
| 
      
 48 
     | 
    
         
            +
             
     | 
| 
      
 49 
     | 
    
         
            +
                  def queue
         
     | 
| 
      
 50 
     | 
    
         
            +
                    class_variable_get(:@@toiler_options)[:queue]
         
     | 
| 
      
 51 
     | 
    
         
            +
                  end
         
     | 
| 
      
 52 
     | 
    
         
            +
             
     | 
| 
      
 53 
     | 
    
         
            +
                  def auto_visibility_timeout?
         
     | 
| 
      
 54 
     | 
    
         
            +
                    class_variable_get(:@@toiler_options)[:auto_visibility_timeout]
         
     | 
| 
      
 55 
     | 
    
         
            +
                  end
         
     | 
| 
      
 56 
     | 
    
         
            +
             
     | 
| 
      
 57 
     | 
    
         
            +
                  def auto_delete?
         
     | 
| 
      
 58 
     | 
    
         
            +
                    class_variable_get(:@@toiler_options)[:auto_delete]
         
     | 
| 
      
 59 
     | 
    
         
            +
                  end
         
     | 
| 
      
 60 
     | 
    
         
            +
                end
         
     | 
| 
      
 61 
     | 
    
         
            +
              end
         
     | 
| 
      
 62 
     | 
    
         
            +
            end
         
     |