toiler 0.3.1.beta1 → 0.3.1.beta2

data/lib/toiler/aws/message.rb

@@ -1,64 +1,64 @@
-module Toiler
-  module Aws
-    # SQS Message abstraction
-    # Provides methods for querying and acting on a SQS message
-    class Message
-      attr_accessor :client, :queue_url, :data
-
-      def initialize(client, queue_url, data)
-        @client = client
-        @queue_url = queue_url
-        @data = data
-      end
-
-      def delete
-        client.delete_message(
-          queue_url: queue_url,
-          receipt_handle: data.receipt_handle
-        )
-      end
-
-      def change_visibility(options)
-        client.change_message_visibility(
-          options.merge(queue_url: queue_url, receipt_handle: receipt_handle)
-        )
-      end
-
-      def visibility_timeout=(timeout)
-        client.change_message_visibility(
-          queue_url: queue_url,
-          receipt_handle: data.receipt_handle,
-          visibility_timeout: timeout
-        )
-      end
-
-      def message_id
-        data.message_id
-      end
-
-      def receipt_handle
-        data.receipt_handle
-      end
-
-      def md5_of_body
-        data.md5_of_body
-      end
-
-      def body
-        data.body
-      end
-
-      def attributes
-        data.attributes
-      end
-
-      def md5_of_message_attributes
-        data.md5_of_message_attributes
-      end
-
-      def message_attributes
-        data.message_attributes
-      end
-    end
-  end
-end
+module Toiler
+  module Aws
+    # SQS Message abstraction
+    # Provides methods for querying and acting on a SQS message
+    class Message
+      attr_accessor :client, :queue_url, :data
+
+      def initialize(client, queue_url, data)
+        @client = client
+        @queue_url = queue_url
+        @data = data
+      end
+
+      def delete
+        client.delete_message(
+          queue_url: queue_url,
+          receipt_handle: data.receipt_handle
+        )
+      end
+
+      def change_visibility(options)
+        client.change_message_visibility(
+          options.merge(queue_url: queue_url, receipt_handle: receipt_handle)
+        )
+      end
+
+      def visibility_timeout=(timeout)
+        client.change_message_visibility(
+          queue_url: queue_url,
+          receipt_handle: data.receipt_handle,
+          visibility_timeout: timeout
+        )
+      end
+
+      def message_id
+        data.message_id
+      end
+
+      def receipt_handle
+        data.receipt_handle
+      end
+
+      def md5_of_body
+        data.md5_of_body
+      end
+
+      def body
+        data.body
+      end
+
+      def attributes
+        data.attributes
+      end
+
+      def md5_of_message_attributes
+        data.md5_of_message_attributes
+      end
+
+      def message_attributes
+        data.message_attributes
+      end
+    end
+  end
+end

data/lib/toiler/aws/queue.rb

@@ -1,61 +1,61 @@
-require 'toiler/aws/message'
-
-module Toiler
-  module Aws
-    # SQS Queue abstraction
-    # Provides methods for querying and acting on a SQS queue
-    class Queue
-      attr_accessor :name, :client, :url
-
-      def initialize(name, client = nil)
-        @name   = name
-        @client = client || ::Aws::SQS::Client.new
-        @url    = client.get_queue_url(queue_name: name).queue_url
-      end
-
-      def visibility_timeout
-        client.get_queue_attributes(
-          queue_url: url,
-          attribute_names: ['VisibilityTimeout']
-        ).attributes['VisibilityTimeout'].to_i
-      end
-
-      def delete_messages(options)
-        client.delete_message_batch options.merge queue_url: url
-      end
-
-      def send_message(options)
-        client.send_message sanitize_message_body options.merge queue_url: url
-      end
-
-      def send_messages(options)
-        client.send_message_batch(
-          sanitize_message_body options.merge queue_url: url
-        )
-      end
-
-      def receive_messages(options)
-        client.receive_message(options.merge(queue_url: url))
-          .messages
-          .map { |m| Message.new(client, url, m) }
-      end
-
-      private
-
-      def sanitize_message_body(options)
-        messages = options[:entries] || [options]
-
-        messages.each do |m|
-          body = m[:message_body]
-          if body.is_a?(Hash)
-            m[:message_body] = JSON.dump(body)
-          elsif !body.is_a? String
-            fail ArgumentError, "Body must be a String, found #{body.class}"
-          end
-        end
-
-        options
-      end
-    end
-  end
-end
+require 'toiler/aws/message'
+
+module Toiler
+  module Aws
+    # SQS Queue abstraction
+    # Provides methods for querying and acting on a SQS queue
+    class Queue
+      attr_accessor :name, :client, :url
+
+      def initialize(name, client = nil)
+        @name   = name
+        @client = client || ::Aws::SQS::Client.new
+        @url    = client.get_queue_url(queue_name: name).queue_url
+      end
+
+      def visibility_timeout
+        client.get_queue_attributes(
+          queue_url: url,
+          attribute_names: ['VisibilityTimeout']
+        ).attributes['VisibilityTimeout'].to_i
+      end
+
+      def delete_messages(options)
+        client.delete_message_batch options.merge queue_url: url
+      end
+
+      def send_message(options)
+        client.send_message sanitize_message_body options.merge queue_url: url
+      end
+
+      def send_messages(options)
+        client.send_message_batch(
+          sanitize_message_body options.merge queue_url: url
+        )
+      end
+
+      def receive_messages(options)
+        client.receive_message(options.merge(queue_url: url))
+          .messages
+          .map { |m| Message.new(client, url, m) }
+      end
+
+      private
+
+      def sanitize_message_body(options)
+        messages = options[:entries] || [options]
+
+        messages.each do |m|
+          body = m[:message_body]
+          if body.is_a?(Hash)
+            m[:message_body] = JSON.dump(body)
+          elsif !body.is_a? String
+            fail ArgumentError, "Body must be a String, found #{body.class}"
+          end
+        end
+
+        options
+      end
+    end
+  end
+end

data/lib/toiler/cli.rb

@@ -1,153 +1,160 @@
-require 'singleton'
-require 'timeout'
-require 'optparse'
-require 'toiler'
-
-module Toiler
-  # See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
-  class Shutdown < Interrupt; end
-
-  # Command line client interface
-  class CLI
-    include Singleton
-
-    attr_accessor :supervisor
-
-    def run(args)
-      @self_read, @self_write = IO.pipe
-
-      trap_signals
-      options = Utils::ArgumentParser.parse(args)
-      Utils::EnvironmentLoader.load(options)
-      daemonize
-      write_pid
-      load_concurrent
-      start_supervisor
-
-      handle_stop
-    end
-
-    private
-
-    def handle_stop
-      while (readable_io = IO.select([@self_read]))
-        handle_signal(readable_io.first[0].gets.strip)
-      end
-    rescue Interrupt
-      puts 'Waiting up to 60 seconds for actors to finish...'
-      supervisor.ask(:terminate!).wait(60)
-    ensure
-      exit 0
-    end
-
-    def shutdown_pools
-      Concurrent.global_fast_executor.shutdown
-      Concurrent.global_io_executor.shutdown
-      return if Concurrent.global_io_executor.wait_for_termination(60)
-      Concurrent.global_io_executor.kill
-    end
-
-    def start_supervisor
-      require 'toiler/actor/supervisor'
-      @supervisor = Actor::Supervisor.spawn! :supervisor
-    end
-
-    def trap_signals
-      %w(INT TERM QUIT USR1 USR2 TTIN).each do |sig|
-        begin
-          trap sig do
-            @self_write.puts(sig)
-          end
-        rescue ArgumentError
-          puts "System does not support signal #{sig}"
-        end
-      end
-    end
-
-    def print_stacktraces
-      return unless Toiler.logger
-      Toiler.logger.info "-------------------\nReceived QUIT, dumping threads:"
-      Thread.list.each do |t|
-        id = t.object_id
-        Toiler.logger.info "[thread:#{id}] #{t.backtrace.join("\n[thread:#{id}] ")}"
-      end
-      Toiler.logger.info '-------------------'
-    end
-
-    def print_status
-      return unless Toiler.logger
-      Toiler.logger.info "-------------------\nReceived QUIT, dumping status:"
-      Toiler.queues.each do |queue|
-        fetcher = Toiler.fetcher(queue).send(:core).send(:context)
-        processor_pool = Toiler.processor_pool(queue).send(:core).send(:context)
-        processors = processor_pool.instance_variable_get(:@workers).collect{|w| w.send(:core).send(:context)}
-        busy_processors = processors.count{|pr| pr.executing?}
-        Toiler.logger.info "[fetcher:#{fetcher.name}] [executing:#{fetcher.executing?}] [scheduled:#{fetcher.scheduled?}] [free_processors:#{fetcher.get_free_processors}]"
-        Toiler.logger.info "[processor_pool:#{processor_pool.name}] [workers:#{processors.count}] [busy:#{busy_processors}]"
-        processors.each do |processor|
-          Toiler.logger.info "[processor:#{processor.name}] [executing:#{processor.executing?}] [thread:#{processor.thread.object_id}]"
-        end
-      end
-      Toiler.logger.info '-------------------'
-    end
-
-    def handle_signal(signal)
-      case signal
-      when 'QUIT'
-        print_stacktraces
-        print_status
-      when 'INT', 'TERM'
-        fail Interrupt
-      end
-    end
-
-    def load_concurrent
-      fail 'Concurrent should not be required now' if defined?(::Concurrent)
-      require 'concurrent-edge'
-      Concurrent.global_logger = lambda do |level, progname, msg = nil, &block|
-        Toiler.logger.log(level, msg, progname, &block)
-      end if Toiler.logger
-    end
-
-    def daemonize
-      return unless Toiler.options[:daemon]
-      fail 'Logfile required when daemonizing' unless Toiler.options[:logfile]
-
-      files_to_reopen = []
-      ObjectSpace.each_object(File) do |file|
-        files_to_reopen << file unless file.closed?
-      end
-
-      Process.daemon(true, true)
-
-      reopen_files(files_to_reopen)
-      reopen_std
-    end
-
-    def reopen_files(files_to_reopen)
-      files_to_reopen.each do |file|
-        begin
-          file.reopen file.path, 'a+'
-          file.sync = true
-        rescue StandardError
-          puts "Failed to reopen file #{file}"
-        end
-      end
-    end
-
-    def reopen_std
-      [$stdout, $stderr].each do |io|
-        File.open(Toiler.options[:logfile], 'ab') do |f|
-          io.reopen(f)
-        end
-        io.sync = true
-      end
-      $stdin.reopen('/dev/null')
-    end
-
-    def write_pid
-      file = Toiler.options[:pidfile]
-      File.write file, Process.pid if file
-    end
-  end
-end
+require 'singleton'
+require 'timeout'
+require 'optparse'
+require 'toiler'
+
+module Toiler
+  # See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
+  class Shutdown < Interrupt; end
+
+  # Command line client interface
+  class CLI
+    include Singleton
+
+    attr_accessor :supervisor
+
+    def run(args)
+      @self_read, @self_write = IO.pipe
+
+      trap_signals
+      options = Utils::ArgumentParser.parse(args)
+      Utils::EnvironmentLoader.load(options)
+      daemonize
+      write_pid
+      load_concurrent
+      start_supervisor
+
+      handle_stop
+    end
+
+    private
+
+    def handle_stop
+      while (readable_io = IO.select([@self_read]))
+        handle_signal(readable_io.first[0].gets.strip)
+      end
+    rescue Interrupt
+      puts 'Waiting up to 60 seconds for actors to finish...'
+      supervisor.ask(:terminate!).wait(60)
+    ensure
+      exit 0
+    end
+
+    def shutdown_pools
+      Concurrent.global_fast_executor.shutdown
+      Concurrent.global_io_executor.shutdown
+      return if Concurrent.global_io_executor.wait_for_termination(60)
+      Concurrent.global_io_executor.kill
+    end
+
+    def start_supervisor
+      require 'toiler/actor/supervisor'
+      @supervisor = Actor::Supervisor.spawn! :supervisor
+    end
+
+    def trap_signals
+      %w(INT TERM QUIT USR1 USR2 TTIN).each do |sig|
+        begin
+          trap sig do
+            @self_write.puts(sig)
+          end
+        rescue ArgumentError
+          puts "System does not support signal #{sig}"
+        end
+      end
+    end
+
+    def print_stacktraces
+      return unless Toiler.logger
+      Toiler.logger.info "-------------------"
+      Toiler.logger.info "Received QUIT, dumping threads:"
+      Thread.list.each do |t|
+        id = t.object_id
+        Toiler.logger.info "[thread:#{id}] #{t.backtrace.join("\n[thread:#{id}] ")}"
+      end
+      Toiler.logger.info '-------------------'
+    end
+
+    def print_status
+      return unless Toiler.logger
+      Toiler.logger.info "-------------------"
+      Toiler.logger.info "Received QUIT, dumping status:"
+      Toiler.queues.each do |queue|
+        fetcher = Toiler.fetcher(queue).send(:core).send(:context)
+        processor_pool = Toiler.processor_pool(queue).send(:core).send(:context)
+        processors = processor_pool.instance_variable_get(:@workers).collect{|w| w.send(:core).send(:context)}
+        busy_processors = processors.count{|pr| pr.executing?}
+        message = "Status for [queue:#{queue}]:"
+        message += "\n[fetcher:#{fetcher.name}] [executing:#{fetcher.executing?}] [polling:#{fetcher.polling?}] [scheduled:#{fetcher.scheduled?}] [free_processors:#{fetcher.get_free_processors}]"
+        message += "\n[processor_pool:#{processor_pool.name}] [workers:#{processors.count}] [busy:#{busy_processors}]"
+        processors.each do |processor|
+          thread = processor.thread
+          thread_id = thread.nil? ? nil : thread.object_id
+          message += "\n[processor:#{processor.name}] [executing:#{processor.executing?}] [thread:#{thread_id}] Stack:"
+          message += thread.backtrace.join("\n\t") unless thread.nil?
+        end
+        Toiler.logger.info message
+      end
+      Toiler.logger.info '-------------------'
+    end
+
+    def handle_signal(signal)
+      case signal
+      when 'QUIT'
+        print_stacktraces
+        print_status
+      when 'INT', 'TERM'
+        fail Interrupt
+      end
+    end
+
+    def load_concurrent
+      fail 'Concurrent should not be required now' if defined?(::Concurrent)
+      require 'concurrent-edge'
+      Concurrent.global_logger = lambda do |level, progname, msg = nil, &block|
+        Toiler.logger.log(level, msg, progname, &block)
+      end if Toiler.logger
+    end
+
+    def daemonize
+      return unless Toiler.options[:daemon]
+      fail 'Logfile required when daemonizing' unless Toiler.options[:logfile]
+
+      files_to_reopen = []
+      ObjectSpace.each_object(File) do |file|
+        files_to_reopen << file unless file.closed?
+      end
+
+      Process.daemon(true, true)
+
+      reopen_files(files_to_reopen)
+      reopen_std
+    end
+
+    def reopen_files(files_to_reopen)
+      files_to_reopen.each do |file|
+        begin
+          file.reopen file.path, 'a+'
+          file.sync = true
+        rescue StandardError
+          puts "Failed to reopen file #{file}"
+        end
+      end
+    end
+
+    def reopen_std
+      [$stdout, $stderr].each do |io|
+        File.open(Toiler.options[:logfile], 'ab') do |f|
+          io.reopen(f)
+        end
+        io.sync = true
+      end
+      $stdin.reopen('/dev/null')
+    end
+
+    def write_pid
+      file = Toiler.options[:pidfile]
+      File.write file, Process.pid if file
+    end
+  end
+end

data/lib/toiler/utils/argument_parser.rb

@@ -1,50 +1,50 @@
-module Toiler
-  module Utils
-    # Parses command-line arguments
-    module ArgumentParser
-      module_function
-
-      def parse(argv)
-        opts = { queues: [] }
-
-        parser = OptionParser.new do |o|
-          o.on '-d', '--daemon', 'Daemonize process' do |arg|
-            opts[:daemon] = arg
-          end
-
-          o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
-            opts[:require] = arg
-          end
-
-          o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
-            opts[:config_file] = arg
-          end
-
-          o.on '-R', '--rails', 'Load Rails' do |arg|
-            opts[:rails] = arg
-          end
-
-          o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
-            opts[:logfile] = arg
-          end
-
-          o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
-            opts[:pidfile] = arg
-          end
-
-          o.on '-v', '--verbose', 'Print more verbose output' do |arg|
-            opts[:verbose] = arg
-          end
-        end
-
-        parser.banner = 'toiler [options]'
-        parser.on_tail '-h', '--help', 'Show help' do
-          puts parser
-          exit 1
-        end
-        parser.parse!(argv)
-        opts
-      end
-    end
-  end
-end
+module Toiler
+  module Utils
+    # Parses command-line arguments
+    module ArgumentParser
+      module_function
+
+      def parse(argv)
+        opts = { queues: [] }
+
+        parser = OptionParser.new do |o|
+          o.on '-d', '--daemon', 'Daemonize process' do |arg|
+            opts[:daemon] = arg
+          end
+
+          o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
+            opts[:require] = arg
+          end
+
+          o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
+            opts[:config_file] = arg
+          end
+
+          o.on '-R', '--rails', 'Load Rails' do |arg|
+            opts[:rails] = arg
+          end
+
+          o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
+            opts[:logfile] = arg
+          end
+
+          o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
+            opts[:pidfile] = arg
+          end
+
+          o.on '-v', '--verbose', 'Print more verbose output' do |arg|
+            opts[:verbose] = arg
+          end
+        end
+
+        parser.banner = 'toiler [options]'
+        parser.on_tail '-h', '--help', 'Show help' do
+          puts parser
+          exit 1
+        end
+        parser.parse!(argv)
+        opts
+      end
+    end
+  end
+end