toiler 0.3.0 → 0.3.1.beta1

data/lib/toiler/actor/supervisor.rb

@@ -1,55 +1,55 @@
-require 'toiler/actor/fetcher'
-require 'toiler/actor/processor'
-
-module Toiler
-  module Actor
-    # Actor that starts and supervises Toiler's actors
-    class Supervisor < Concurrent::Actor::RestartingContext
-      include Utils::ActorLogging
-
-      attr_accessor :client
-
-      def initialize
-        @client = ::Aws::SQS::Client.new
-        spawn_fetchers
-        spawn_processors
-      end
-
-      def on_message(_msg)
-        pass
-      end
-
-      def queues
-        Toiler.worker_class_registry
-      end
-
-      def spawn_fetchers
-        queues.each do |queue, _klass|
-          begin
-            fetcher = Actor::Fetcher.spawn! name: "fetcher_#{queue}".to_sym,
-                                            supervise: true, args: [queue, client]
-            Toiler.set_fetcher queue, fetcher
-          rescue StandardError => e
-            error "Failed to start Fetcher for queue #{queue}: #{e.message}\n#{e.backtrace.join("\n")}"
-          end
-        end
-      end
-
-      def spawn_processors
-        queues.each do |queue, klass|
-          name = "processor_pool_#{queue}".to_sym
-          count = klass.concurrency
-          begin
-            pool = Concurrent::Actor::Utils::Pool.spawn! name, count do |index|
-                Actor::Processor.spawn name: "processor_#{queue}_#{index}".to_sym,
-                                       supervise: true, args: [queue]
-            end
-            Toiler.set_processor_pool queue, pool
-          rescue StandardError => e
-            error "Failed to spawn Processor Pool for queue #{queue}: #{e.message}\n#{e.backtrace.join("\n")}"
-          end
-        end
-      end
-    end
-  end
-end
+require 'toiler/actor/fetcher'
+require 'toiler/actor/processor'
+
+module Toiler
+  module Actor
+    # Actor that starts and supervises Toiler's actors
+    class Supervisor < Concurrent::Actor::RestartingContext
+      include Utils::ActorLogging
+
+      attr_accessor :client
+
+      def initialize
+        @client = ::Aws::SQS::Client.new
+        spawn_fetchers
+        spawn_processors
+      end
+
+      def on_message(_msg)
+        pass
+      end
+
+      def queues
+        Toiler.worker_class_registry
+      end
+
+      def spawn_fetchers
+        queues.each do |queue, _klass|
+          begin
+            fetcher = Actor::Fetcher.spawn! name: "fetcher_#{queue}".to_sym,
+                                            supervise: true, args: [queue, client]
+            Toiler.set_fetcher queue, fetcher
+          rescue StandardError => e
+            error "Failed to start Fetcher for queue #{queue}: #{e.message}\n#{e.backtrace.join("\n")}"
+          end
+        end
+      end
+
+      def spawn_processors
+        queues.each do |queue, klass|
+          name = "processor_pool_#{queue}".to_sym
+          count = klass.concurrency
+          begin
+            pool = Concurrent::Actor::Utils::Pool.spawn! name, count do |index|
+                Actor::Processor.spawn name: "processor_#{queue}_#{index}".to_sym,
+                                       supervise: true, args: [queue]
+            end
+            Toiler.set_processor_pool queue, pool
+          rescue StandardError => e
+            error "Failed to spawn Processor Pool for queue #{queue}: #{e.message}\n#{e.backtrace.join("\n")}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/toiler/actor/utils/actor_logging.rb

@@ -1,28 +1,28 @@
-module Toiler
-  module Actor
-    module Utils
-      # Provides helper methods for logging
-      module ActorLogging
-        def error(msg)
-          log Logger::Severity::ERROR,  msg
-        end
-
-        def info(msg)
-          log Logger::Severity::INFO, msg
-        end
-
-        def debug(msg)
-          log Logger::Severity::DEBUG,  msg
-        end
-
-        def warn(msg)
-          log Logger::Severity::WARN, smsg
-        end
-
-        def fatal(msg)
-          log Logger::Severity::FATAL,  msg
-        end
-      end
-    end
-  end
-end
+module Toiler
+  module Actor
+    module Utils
+      # Provides helper methods for logging
+      module ActorLogging
+        def error(msg)
+          log Logger::Severity::ERROR,  msg
+        end
+
+        def info(msg)
+          log Logger::Severity::INFO, msg
+        end
+
+        def debug(msg)
+          log Logger::Severity::DEBUG,  msg
+        end
+
+        def warn(msg)
+          log Logger::Severity::WARN, smsg
+        end
+
+        def fatal(msg)
+          log Logger::Severity::FATAL,  msg
+        end
+      end
+    end
+  end
+end

data/lib/toiler/aws/message.rb

@@ -1,64 +1,64 @@
-module Toiler
-  module Aws
-    # SQS Message abstraction
-    # Provides methods for querying and acting on a SQS message
-    class Message
-      attr_accessor :client, :queue_url, :data
-
-      def initialize(client, queue_url, data)
-        @client = client
-        @queue_url = queue_url
-        @data = data
-      end
-
-      def delete
-        client.delete_message(
-          queue_url: queue_url,
-          receipt_handle: data.receipt_handle
-        )
-      end
-
-      def change_visibility(options)
-        client.change_message_visibility(
-          options.merge(queue_url: queue_url, receipt_handle: receipt_handle)
-        )
-      end
-
-      def visibility_timeout=(timeout)
-        client.change_message_visibility(
-          queue_url: queue_url,
-          receipt_handle: data.receipt_handle,
-          visibility_timeout: timeout
-        )
-      end
-
-      def message_id
-        data.message_id
-      end
-
-      def receipt_handle
-        data.receipt_handle
-      end
-
-      def md5_of_body
-        data.md5_of_body
-      end
-
-      def body
-        data.body
-      end
-
-      def attributes
-        data.attributes
-      end
-
-      def md5_of_message_attributes
-        data.md5_of_message_attributes
-      end
-
-      def message_attributes
-        data.message_attributes
-      end
-    end
-  end
-end
+module Toiler
+  module Aws
+    # SQS Message abstraction
+    # Provides methods for querying and acting on a SQS message
+    class Message
+      attr_accessor :client, :queue_url, :data
+
+      def initialize(client, queue_url, data)
+        @client = client
+        @queue_url = queue_url
+        @data = data
+      end
+
+      def delete
+        client.delete_message(
+          queue_url: queue_url,
+          receipt_handle: data.receipt_handle
+        )
+      end
+
+      def change_visibility(options)
+        client.change_message_visibility(
+          options.merge(queue_url: queue_url, receipt_handle: receipt_handle)
+        )
+      end
+
+      def visibility_timeout=(timeout)
+        client.change_message_visibility(
+          queue_url: queue_url,
+          receipt_handle: data.receipt_handle,
+          visibility_timeout: timeout
+        )
+      end
+
+      def message_id
+        data.message_id
+      end
+
+      def receipt_handle
+        data.receipt_handle
+      end
+
+      def md5_of_body
+        data.md5_of_body
+      end
+
+      def body
+        data.body
+      end
+
+      def attributes
+        data.attributes
+      end
+
+      def md5_of_message_attributes
+        data.md5_of_message_attributes
+      end
+
+      def message_attributes
+        data.message_attributes
+      end
+    end
+  end
+end

data/lib/toiler/aws/queue.rb

@@ -1,61 +1,61 @@
-require 'toiler/aws/message'
-
-module Toiler
-  module Aws
-    # SQS Queue abstraction
-    # Provides methods for querying and acting on a SQS queue
-    class Queue
-      attr_accessor :name, :client, :url
-
-      def initialize(name, client = nil)
-        @name   = name
-        @client = client || ::Aws::SQS::Client.new
-        @url    = client.get_queue_url(queue_name: name).queue_url
-      end
-
-      def visibility_timeout
-        client.get_queue_attributes(
-          queue_url: url,
-          attribute_names: ['VisibilityTimeout']
-        ).attributes['VisibilityTimeout'].to_i
-      end
-
-      def delete_messages(options)
-        client.delete_message_batch options.merge queue_url: url
-      end
-
-      def send_message(options)
-        client.send_message sanitize_message_body options.merge queue_url: url
-      end
-
-      def send_messages(options)
-        client.send_message_batch(
-          sanitize_message_body options.merge queue_url: url
-        )
-      end
-
-      def receive_messages(options)
-        client.receive_message(options.merge(queue_url: url))
-          .messages
-          .map { |m| Message.new(client, url, m) }
-      end
-
-      private
-
-      def sanitize_message_body(options)
-        messages = options[:entries] || [options]
-
-        messages.each do |m|
-          body = m[:message_body]
-          if body.is_a?(Hash)
-            m[:message_body] = JSON.dump(body)
-          elsif !body.is_a? String
-            fail ArgumentError, "Body must be a String, found #{body.class}"
-          end
-        end
-
-        options
-      end
-    end
-  end
-end
+require 'toiler/aws/message'
+
+module Toiler
+  module Aws
+    # SQS Queue abstraction
+    # Provides methods for querying and acting on a SQS queue
+    class Queue
+      attr_accessor :name, :client, :url
+
+      def initialize(name, client = nil)
+        @name   = name
+        @client = client || ::Aws::SQS::Client.new
+        @url    = client.get_queue_url(queue_name: name).queue_url
+      end
+
+      def visibility_timeout
+        client.get_queue_attributes(
+          queue_url: url,
+          attribute_names: ['VisibilityTimeout']
+        ).attributes['VisibilityTimeout'].to_i
+      end
+
+      def delete_messages(options)
+        client.delete_message_batch options.merge queue_url: url
+      end
+
+      def send_message(options)
+        client.send_message sanitize_message_body options.merge queue_url: url
+      end
+
+      def send_messages(options)
+        client.send_message_batch(
+          sanitize_message_body options.merge queue_url: url
+        )
+      end
+
+      def receive_messages(options)
+        client.receive_message(options.merge(queue_url: url))
+          .messages
+          .map { |m| Message.new(client, url, m) }
+      end
+
+      private
+
+      def sanitize_message_body(options)
+        messages = options[:entries] || [options]
+
+        messages.each do |m|
+          body = m[:message_body]
+          if body.is_a?(Hash)
+            m[:message_body] = JSON.dump(body)
+          elsif !body.is_a? String
+            fail ArgumentError, "Body must be a String, found #{body.class}"
+          end
+        end
+
+        options
+      end
+    end
+  end
+end

data/lib/toiler/cli.rb

@@ -1,135 +1,153 @@
-require 'singleton'
-require 'timeout'
-require 'optparse'
-require 'toiler'
-
-module Toiler
-  # See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
-  class Shutdown < Interrupt; end
-
-  # Command line client interface
-  class CLI
-    include Singleton
-
-    attr_accessor :supervisor
-
-    def run(args)
-      @self_read, @self_write = IO.pipe
-
-      trap_signals
-      options = Utils::ArgumentParser.parse(args)
-      Utils::EnvironmentLoader.load(options)
-      daemonize
-      write_pid
-      load_concurrent
-      start_supervisor
-
-      handle_stop
-    end
-
-    private
-
-    def handle_stop
-      while (readable_io = IO.select([@self_read]))
-        handle_signal(readable_io.first[0].gets.strip)
-      end
-    rescue Interrupt
-      puts 'Waiting up to 60 seconds for actors to finish...'
-      supervisor.ask(:terminate!).wait(60)
-    ensure
-      exit 0
-    end
-
-    def shutdown_pools
-      Concurrent.global_fast_executor.shutdown
-      Concurrent.global_io_executor.shutdown
-      return if Concurrent.global_io_executor.wait_for_termination(60)
-      Concurrent.global_io_executor.kill
-    end
-
-    def start_supervisor
-      require 'toiler/actor/supervisor'
-      @supervisor = Actor::Supervisor.spawn! :supervisor
-    end
-
-    def trap_signals
-      %w(INT TERM QUIT USR1 USR2 TTIN).each do |sig|
-        begin
-          trap sig do
-            @self_write.puts(sig)
-          end
-        rescue ArgumentError
-          puts "System does not support signal #{sig}"
-        end
-      end
-    end
-
-    def print_stacktraces
-      return unless Toiler.logger
-      Toiler.logger.info "-------------------\nReceived QUIT, dumping threads:"
-      Thread.list.each do |t|
-        id = t.object_id
-        Toiler.logger.info "[thread:#{id}] #{t.backtrace.join("\n[thread:#{id}] ")}"
-      end
-      Toiler.logger.info '-------------------'
-    end
-
-    def handle_signal(signal)
-      case signal
-      when 'QUIT'
-        print_stacktraces
-      when 'INT', 'TERM'
-        fail Interrupt
-      end
-    end
-
-    def load_concurrent
-      fail 'Concurrent should not be required now' if defined?(::Concurrent)
-      require 'concurrent-edge'
-      Concurrent.global_logger = lambda do |level, progname, msg = nil, &block|
-        Toiler.logger.log(level, msg, progname, &block)
-      end if Toiler.logger
-    end
-
-    def daemonize
-      return unless Toiler.options[:daemon]
-      fail 'Logfile required when daemonizing' unless Toiler.options[:logfile]
-
-      files_to_reopen = []
-      ObjectSpace.each_object(File) do |file|
-        files_to_reopen << file unless file.closed?
-      end
-
-      Process.daemon(true, true)
-
-      reopen_files(files_to_reopen)
-      reopen_std
-    end
-
-    def reopen_files(files_to_reopen)
-      files_to_reopen.each do |file|
-        begin
-          file.reopen file.path, 'a+'
-          file.sync = true
-        rescue StandardError
-          puts "Failed to reopen file #{file}"
-        end
-      end
-    end
-
-    def reopen_std
-      [$stdout, $stderr].each do |io|
-        File.open(Toiler.options[:logfile], 'ab') do |f|
-          io.reopen(f)
-        end
-        io.sync = true
-      end
-      $stdin.reopen('/dev/null')
-    end
-
-    def write_pid
-      file = Toiler.options[:pidfile]
-      File.write file, Process.pid if file
-    end
-  end
-end
+require 'singleton'
+require 'timeout'
+require 'optparse'
+require 'toiler'
+
+module Toiler
+  # See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
+  class Shutdown < Interrupt; end
+
+  # Command line client interface
+  class CLI
+    include Singleton
+
+    attr_accessor :supervisor
+
+    def run(args)
+      @self_read, @self_write = IO.pipe
+
+      trap_signals
+      options = Utils::ArgumentParser.parse(args)
+      Utils::EnvironmentLoader.load(options)
+      daemonize
+      write_pid
+      load_concurrent
+      start_supervisor
+
+      handle_stop
+    end
+
+    private
+
+    def handle_stop
+      while (readable_io = IO.select([@self_read]))
+        handle_signal(readable_io.first[0].gets.strip)
+      end
+    rescue Interrupt
+      puts 'Waiting up to 60 seconds for actors to finish...'
+      supervisor.ask(:terminate!).wait(60)
+    ensure
+      exit 0
+    end
+
+    def shutdown_pools
+      Concurrent.global_fast_executor.shutdown
+      Concurrent.global_io_executor.shutdown
+      return if Concurrent.global_io_executor.wait_for_termination(60)
+      Concurrent.global_io_executor.kill
+    end
+
+    def start_supervisor
+      require 'toiler/actor/supervisor'
+      @supervisor = Actor::Supervisor.spawn! :supervisor
+    end
+
+    def trap_signals
+      %w(INT TERM QUIT USR1 USR2 TTIN).each do |sig|
+        begin
+          trap sig do
+            @self_write.puts(sig)
+          end
+        rescue ArgumentError
+          puts "System does not support signal #{sig}"
+        end
+      end
+    end
+
+    def print_stacktraces
+      return unless Toiler.logger
+      Toiler.logger.info "-------------------\nReceived QUIT, dumping threads:"
+      Thread.list.each do |t|
+        id = t.object_id
+        Toiler.logger.info "[thread:#{id}] #{t.backtrace.join("\n[thread:#{id}] ")}"
+      end
+      Toiler.logger.info '-------------------'
+    end
+
+    def print_status
+      return unless Toiler.logger
+      Toiler.logger.info "-------------------\nReceived QUIT, dumping status:"
+      Toiler.queues.each do |queue|
+        fetcher = Toiler.fetcher(queue).send(:core).send(:context)
+        processor_pool = Toiler.processor_pool(queue).send(:core).send(:context)
+        processors = processor_pool.instance_variable_get(:@workers).collect{|w| w.send(:core).send(:context)}
+        busy_processors = processors.count{|pr| pr.executing?}
+        Toiler.logger.info "[fetcher:#{fetcher.name}] [executing:#{fetcher.executing?}] [scheduled:#{fetcher.scheduled?}] [free_processors:#{fetcher.get_free_processors}]"
+        Toiler.logger.info "[processor_pool:#{processor_pool.name}] [workers:#{processors.count}] [busy:#{busy_processors}]"
+        processors.each do |processor|
+          Toiler.logger.info "[processor:#{processor.name}] [executing:#{processor.executing?}] [thread:#{processor.thread.object_id}]"
+        end
+      end
+      Toiler.logger.info '-------------------'
+    end
+
+    def handle_signal(signal)
+      case signal
+      when 'QUIT'
+        print_stacktraces
+        print_status
+      when 'INT', 'TERM'
+        fail Interrupt
+      end
+    end
+
+    def load_concurrent
+      fail 'Concurrent should not be required now' if defined?(::Concurrent)
+      require 'concurrent-edge'
+      Concurrent.global_logger = lambda do |level, progname, msg = nil, &block|
+        Toiler.logger.log(level, msg, progname, &block)
+      end if Toiler.logger
+    end
+
+    def daemonize
+      return unless Toiler.options[:daemon]
+      fail 'Logfile required when daemonizing' unless Toiler.options[:logfile]
+
+      files_to_reopen = []
+      ObjectSpace.each_object(File) do |file|
+        files_to_reopen << file unless file.closed?
+      end
+
+      Process.daemon(true, true)
+
+      reopen_files(files_to_reopen)
+      reopen_std
+    end
+
+    def reopen_files(files_to_reopen)
+      files_to_reopen.each do |file|
+        begin
+          file.reopen file.path, 'a+'
+          file.sync = true
+        rescue StandardError
+          puts "Failed to reopen file #{file}"
+        end
+      end
+    end
+
+    def reopen_std
+      [$stdout, $stderr].each do |io|
+        File.open(Toiler.options[:logfile], 'ab') do |f|
+          io.reopen(f)
+        end
+        io.sync = true
+      end
+      $stdin.reopen('/dev/null')
+    end
+
+    def write_pid
+      file = Toiler.options[:pidfile]
+      File.write file, Process.pid if file
+    end
+  end
+end