toiler 0.1.5 → 0.2.0

data/lib/toiler/actor/supervisor.rb

@@ -0,0 +1,45 @@
+require 'toiler/actor/fetcher'
+require 'toiler/actor/processor'
+
+module Toiler
+  module Actor
+    # Actor that starts and supervises Toiler's actors
+    class Supervisor < Concurrent::Actor::RestartingContext
+      attr_accessor :client
+
+      def initialize
+        @client = ::Aws::SQS::Client.new
+        spawn_fetchers
+        spawn_processors
+      end
+
+      def on_message(_msg)
+        pass
+      end
+
+      def queues
+        Toiler.worker_class_registry
+      end
+
+      def spawn_fetchers
+        queues.each do |queue, _klass|
+          fetcher = Actor::Fetcher.spawn! name: "fetcher_#{queue}".to_sym,
+                                          supervise: true, args: [queue, client]
+          Toiler.set_fetcher queue, fetcher
+        end
+      end
+
+      def spawn_processors
+        queues.each do |queue, klass|
+          name = "processor_pool_#{queue}".to_sym
+          count = klass.concurrency
+          pool = Concurrent::Actor::Utils::Pool.spawn! name, count do |index|
+            Actor::Processor.spawn name: "processor_#{queue}_#{index}".to_sym,
+                                   supervise: true, args: [queue]
+          end
+          Toiler.set_processor_pool queue, pool
+        end
+      end
+    end
+  end
+end

data/lib/toiler/actor/utils/actor_logging.rb

@@ -0,0 +1,28 @@
+module Toiler
+  module Actor
+    module Utils
+      # Provides helper methods for logging
+      module ActorLogging
+        def error(msg)
+          log Logger::Severity::ERROR, self.class, msg
+        end
+
+        def info(msg)
+          log Logger::Severity::INFO, self.class, msg
+        end
+
+        def debug(msg)
+          log Logger::Severity::DEBUG, self.class, msg
+        end
+
+        def warn(msg)
+          log Logger::Severity::WARN, self.class, msg
+        end
+
+        def fatal(msg)
+          log Logger::Severity::FATAL, self.class, msg
+        end
+      end
+    end
+  end
+end

data/lib/toiler/aws/message.rb

@@ -0,0 +1,64 @@
+module Toiler
+  module Aws
+    # SQS Message abstraction
+    # Provides methods for querying and acting on a SQS message
+    class Message
+      attr_accessor :client, :queue_url, :data
+
+      def initialize(client, queue_url, data)
+        @client = client
+        @queue_url = queue_url
+        @data = data
+      end
+
+      def delete
+        client.delete_message(
+          queue_url: queue_url,
+          receipt_handle: data.receipt_handle
+        )
+      end
+
+      def change_visibility(options)
+        client.change_message_visibility(
+          options.merge(queue_url: queue_url, receipt_handle: receipt_handle)
+        )
+      end
+
+      def visibility_timeout=(timeout)
+        client.change_message_visibility(
+          queue_url: queue_url,
+          receipt_handle: data.receipt_handle,
+          visibility_timeout: timeout
+        )
+      end
+
+      def message_id
+        data.message_id
+      end
+
+      def receipt_handle
+        data.receipt_handle
+      end
+
+      def md5_of_body
+        data.md5_of_body
+      end
+
+      def body
+        data.body
+      end
+
+      def attributes
+        data.attributes
+      end
+
+      def md5_of_message_attributes
+        data.md5_of_message_attributes
+      end
+
+      def message_attributes
+        data.message_attributes
+      end
+    end
+  end
+end

data/lib/toiler/aws/queue.rb

@@ -0,0 +1,61 @@
+require 'toiler/aws/message'
+
+module Toiler
+  module Aws
+    # SQS Queue abstraction
+    # Provides methods for querying and acting on a SQS queue
+    class Queue
+      attr_accessor :name, :client, :url
+
+      def initialize(name, client = nil)
+        @name   = name
+        @client = client || ::Aws::SQS::Client.new
+        @url    = client.get_queue_url(queue_name: name).queue_url
+      end
+
+      def visibility_timeout
+        client.get_queue_attributes(
+          queue_url: url,
+          attribute_names: ['VisibilityTimeout']
+        ).attributes['VisibilityTimeout'].to_i
+      end
+
+      def delete_messages(options)
+        client.delete_message_batch options.merge queue_url: url
+      end
+
+      def send_message(options)
+        client.send_message sanitize_message_body options.merge queue_url: url
+      end
+
+      def send_messages(options)
+        client.send_message_batch(
+          sanitize_message_body options.merge queue_url: url
+        )
+      end
+
+      def receive_messages(options)
+        client.receive_message(options.merge(queue_url: url))
+          .messages
+          .map { |m| Message.new(client, url, m) }
+      end
+
+      private
+
+      def sanitize_message_body(options)
+        messages = options[:entries] || [options]
+
+        messages.each do |m|
+          body = m[:message_body]
+          if body.is_a?(Hash)
+            m[:message_body] = JSON.dump(body)
+          elsif !body.is_a? String
+            fail ArgumentError, "Body must be a String, found #{body.class}"
+          end
+        end
+
+        options
+      end
+    end
+  end
+end

data/lib/toiler/cli.rb

@@ -7,71 +7,81 @@ module Toiler
   # See: https://github.com/mperham/sidekiq/blob/33f5d6b2b6c0dfaab11e5d39688cab7ebadc83ae/lib/sidekiq/cli.rb#L20
   class Shutdown < Interrupt; end
 
+  # Command line client interface
   class CLI
     include Singleton
 
-    def run(args)
-      self_read, self_write = IO.pipe
-
-      %w(INT TERM USR1 USR2 TTIN).each do |sig|
-        begin
-          trap sig do
-            self_write.puts(sig)
-          end
-        rescue ArgumentError
-          puts "Signal #{sig} not supported"
-        end
-      end
+    attr_accessor :supervisor
 
-      options = parse_cli_args(args)
+    def run(args)
+      @self_read, @self_write = IO.pipe
 
-      EnvironmentLoader.load(options)
+      trap_signals
+      options = Utils::ArgumentParser.parse(args)
+      Utils::EnvironmentLoader.load(options)
       daemonize
       write_pid
-      load_celluloid
+      load_concurrent
+      start_supervisor
 
-      begin
-        require 'toiler/supervisor'
-        @supervisor = Supervisor.new
+      handle_stop
+    end
 
-        while (readable_io = IO.select([self_read]))
-          signal = readable_io.first[0].gets.strip
-          handle_signal(signal)
-        end
-      rescue Interrupt
-        puts 'Received interrupt, terminating actors...'
+    private
+
+    def handle_stop
+      while (readable_io = IO.select([@self_read]))
+        handle_signal(readable_io.first[0].gets.strip)
+      end
+    rescue Interrupt
+      puts 'Waiting up to 60 seconds for actors to finish...'
+      supervisor.ask(:terminate!).wait(60)
+    ensure
+      exit 0
+    end
+
+    def shutdown_pools
+      Concurrent.global_fast_executor.shutdown
+      Concurrent.global_io_executor.shutdown
+      return if Concurrent.global_io_executor.wait_for_termination(60)
+      Concurrent.global_io_executor.kill
+    end
+
+    def start_supervisor
+      require 'toiler/actor/supervisor'
+      @supervisor = Actor::Supervisor.spawn! :supervisor
+    end
+
+    def trap_signals
+      %w(INT TERM USR1 USR2 TTIN).each do |sig|
         begin
-          Timeout.timeout(20) do
-            @supervisor.stop
+          trap sig do
+            @self_write.puts(sig)
           end
-        ensure
-          exit 0
+        rescue ArgumentError
+          puts "System does not support signal #{sig}"
         end
       end
     end
 
-    private
-
-    def handle_signal(_signal)
-      fail Interrupt
+    def handle_signal(signal)
+      case signal
+      when 'INT', 'TERM'
+        fail Interrupt
+      end
     end
 
-    def load_celluloid
-      fail "Celluloid cannot be required until here, or it will break Toiler's daemonization" if defined?(::Celluloid) && Toiler.options[:daemon]
-
-      # Celluloid can't be loaded until after we've daemonized
-      # because it spins up threads and creates locks which get
-      # into a very bad state if forked.
-      require 'celluloid/current'
-      require 'celluloid/task/pooled_fiber'
-      Celluloid.task_class = Celluloid::Task::PooledFiber
-      Celluloid.logger = (Toiler.options[:verbose] ? Toiler.logger : nil)
+    def load_concurrent
+      fail 'Concurrent should not be required now' if defined?(::Concurrent)
+      require 'concurrent-edge'
+      Concurrent.global_logger = lambda do |level, progname, msg = nil, &block|
+        Toiler.logger.log(level, msg, progname, &block)
+      end if Toiler.logger
     end
 
     def daemonize
       return unless Toiler.options[:daemon]
-
-      fail ArgumentError, "You really should set a logfile if you're going to daemonize" unless Toiler.options[:logfile]
+      fail 'Logfile required when daemonizing' unless Toiler.options[:logfile]
 
       files_to_reopen = []
       ObjectSpace.each_object(File) do |file|
@@ -80,71 +90,34 @@ module Toiler
 
       Process.daemon(true, true)
 
+      reopen_files(files_to_reopen)
+      reopen_std
+    end
+
+    def reopen_files(files_to_reopen)
       files_to_reopen.each do |file|
         begin
           file.reopen file.path, 'a+'
-          # file.sync = true
-        rescue ::Exception
+          file.sync = true
+        rescue StandardError
+          puts "Failed to reopen file #{file}"
         end
       end
+    end
 
+    def reopen_std
       [$stdout, $stderr].each do |io|
         File.open(Toiler.options[:logfile], 'ab') do |f|
           io.reopen(f)
         end
-        # io.sync = true
+        io.sync = true
       end
       $stdin.reopen('/dev/null')
     end
 
     def write_pid
-      if (path = Toiler.options[:pidfile])
-        File.open(path, 'w') do |f|
-          f.puts Process.pid
-        end
-      end
-    end
-
-    def parse_cli_args(argv)
-      opts = { queues: [] }
-
-      @parser = OptionParser.new do |o|
-        o.on '-d', '--daemon', 'Daemonize process' do |arg|
-          opts[:daemon] = arg
-        end
-
-        o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
-          opts[:require] = arg
-        end
-
-        o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
-          opts[:config_file] = arg
-        end
-
-        o.on '-R', '--rails', 'Load Rails' do |arg|
-          opts[:rails] = arg
-        end
-
-        o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
-          opts[:logfile] = arg
-        end
-
-        o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
-          opts[:pidfile] = arg
-        end
-
-        o.on '-v', '--verbose', 'Print more verbose output' do |arg|
-          opts[:verbose] = arg
-        end
-      end
-
-      @parser.banner = 'toiler [options]'
-      @parser.on_tail '-h', '--help', 'Show help' do
-        Toiler.logger.info @parser
-        exit 1
-      end
-      @parser.parse!(argv)
-      opts
+      file = Toiler.options[:pidfile]
+      File.write file, Process.pid if file
     end
   end
 end

data/lib/toiler/utils/argument_parser.rb

@@ -0,0 +1,50 @@
+module Toiler
+  module Utils
+    # Parses command-line arguments
+    module ArgumentParser
+      module_function
+
+      def parse(argv)
+        opts = { queues: [] }
+
+        parser = OptionParser.new do |o|
+          o.on '-d', '--daemon', 'Daemonize process' do |arg|
+            opts[:daemon] = arg
+          end
+
+          o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
+            opts[:require] = arg
+          end
+
+          o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
+            opts[:config_file] = arg
+          end
+
+          o.on '-R', '--rails', 'Load Rails' do |arg|
+            opts[:rails] = arg
+          end
+
+          o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
+            opts[:logfile] = arg
+          end
+
+          o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
+            opts[:pidfile] = arg
+          end
+
+          o.on '-v', '--verbose', 'Print more verbose output' do |arg|
+            opts[:verbose] = arg
+          end
+        end
+
+        parser.banner = 'toiler [options]'
+        parser.on_tail '-h', '--help', 'Show help' do
+          puts parser
+          exit 1
+        end
+        parser.parse!(argv)
+        opts
+      end
+    end
+  end
+end