sidekiq-cluster 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2c5be67c437089d629dd84e49d3370f58ffd337d
-  data.tar.gz: ea264f35a67a922097e22c432381b95573c2796f
+  metadata.gz: 0cee3d1d2e77f76ea89ce26c50bb43346cf55ecd
+  data.tar.gz: 36cce663074cfda880ebf60e5de3e450786d8c4d
 SHA512:
-  metadata.gz: 2f8062fd63c9e0a7c7bb6dff4927e24e98cc61004454acd35abd5b32a2ea0a4b996d9771c89633502d0a914d592d0f7351b27bb01bcaa5fd9d0acd8ab524a474
-  data.tar.gz: abb384d6f36a0994d5d69b2e66c2c6a1a6a6a169364c3da853f66e663872bfdc8166144b16a5de96674babc422d9f855a9af57c4cbe3b0f1c27a96eb45c1eab6
+  metadata.gz: 58a069bb53080b168197b9ce9f1e4cf574fec259fb3f6fbd25fbb383b08d3844ebaab9566c5265d2aa91d5fa3e5af6c680c134247e57a498417378416c4b82f0
+  data.tar.gz: 8d041f63d583fb3d3a5108944f7ef1d65978d71e1ee3ecca45968737e032e17b81350e5ec7f24768cfb0d8593ab96bf79ad429aa5ef136e582861ce3654d8802

data/exe/sidekiq-cluster

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
 require_relative '../lib/sidekiq/cluster/cli'
-Sidekiq::Cluster::CLI.new(ARGV).run!
+Sidekiq::Cluster::CLI.new(ARGV).execute!
 
 

data/lib/sidekiq/cluster.rb

@@ -1,4 +1,4 @@
-require 'sidekiq/cluster/version'
+require_relative 'cluster/version'
 
 module Sidekiq
   module Cluster
@@ -6,4 +6,7 @@ module Sidekiq
 end
 
 require_relative 'cluster/cli'
+require_relative 'cluster/config'
+require_relative 'cluster/memory'
+require_relative 'cluster/stdlib_ext'
 

data/lib/sidekiq/cluster/cli.rb

@@ -3,273 +3,193 @@ require 'etc'
 require 'optparse'
 require 'logger'
 require 'colored2'
+require 'pp'
+
+require_relative 'config'
+require_relative 'worker_pool'
+require_relative 'version'
 
 module Sidekiq
   module Cluster
-    MAX_RAM_PCT = 80
-
-    ProcessDescriptor = Struct.new(:pid, :index, :pidfile)
-
     class CLI
-      attr_accessor :name,
-                    :pid_prefix,
-                    :argv,
-                    :sidekiq_argv,
-                    :process_count,
-                    :pids,
-                    :processes,
-                    :watch_children,
-                    :memory_percentage_limit,
-                    :log_device,
-                    :logger
-
-      def initialize(argv = ARGV.dup)
-        self.argv                    = argv
-        self.name                    = 'default'
-        self.pid_prefix              = '/tmp/sidekiq.pid'
-        self.sidekiq_argv            = []
-        self.log_device              = STDOUT
-        self.memory_percentage_limit = MAX_RAM_PCT
-        self.process_count           = Etc.nprocessors - 1
-        self.processes               = {}
-        self.pids                    = []
-      end
+      attr_accessor :argv, :config, :logger, :log_device, :worker_pool, :master_pid
+      attr_reader :stdout, :stdin, :stderr, :kernel
 
-      def per_process_memory_limit
-        memory_percentage_limit.to_f / process_count.to_f
+      class << self
+        attr_accessor :instance
       end
 
-      def run!
-        initialize_cli_arguments!
-        print_header!
-        start_main_loop!
+      def initialize(argv, stdin = STDIN, stdout = STDOUT, stderr = STDERR, kernel = Kernel)
+        @argv, @stdin, @stdout, @stderr, @kernel = argv, stdin, stdout, stderr, kernel
+        args                                     = argv.dup
+        args                                     = %w(--help) if args.nil? || args.empty?
+        self.master_pid                          = Process.pid
+        self.config                              = Config.config
+        Config.split_argv(args.dup)
+        self.class.instance = self
       end
 
-      def start_main_loop!
-        start_children.each do |descriptor|
-          processes[descriptor.pid] = descriptor
-        end
-
-        self.pids           = processes.keys
-        self.watch_children = true
+      def execute!
+        parse_cli_arguments
+        print_header
+        pp config.to_h if config.debug
 
-        main
-        setup_signal_traps
-
-        Process.waitall
-        info 'shutting down...'
-      end
+        ::Process.setproctitle('sidekiq-cluster.' + config.name + ' ' + config.cluster_argv.join(' '))
 
-      def print(*args)
-        puts(*args)
-      end
-
-      def stop!(code = 0)
-        Kernel.exit(code)
-      end
-
-      def initialize_cli_arguments!
-        if argv.nil? || argv.empty?
-          argv << '-h'
-        else
-          split_argv! if argv.include?('--')
+        if config.work_dir
+          if Dir.exist?(config.work_dir)
+            Dir.chdir(config.work_dir)
+          else
+            raise "Can not find work dir #{config.work_dir}"
+          end
         end
-        init_logger!
-        parse_args!
-      end
 
-      private
+        config.logfile    = Dir.pwd + '/' + config.logfile if config.logfile && !config.logfile.start_with?('/')
+        config.pid_prefix = Dir.pwd + '/' + config.pid_prefix if config.pid_prefix && !config.pid_prefix.start_with?('/')
 
-      def print_header!
-        info "starting up sidekiq-cluster for #{name}"
-        info "NOTE: cluster max memory limit is #{memory_percentage_limit.round(2)}% of total"
-        info "NOTE: per sidekiq memory limit is #{per_process_memory_limit.round(2)}% of total"
-      end
+        init_logger
 
-      def init_logger!
-        self.logger = ::Logger.new(log_device, level: ::Logger::INFO)
+        boot
       end
 
-      def split_argv!
-        self.sidekiq_argv = argv[(argv.index('--') + 1)..-1]
-        self.argv         = argv[0..(argv.index('--') - 1)]
+      def boot
+        @worker_pool = ::Sidekiq::Cluster::WorkerPool.new(self, config)
+        @worker_pool.spawn
       end
 
-      def setup_signal_traps
-        %w(INT USR1 TERM).each do |sig|
-          Signal.trap(sig) do
-            handle_sig(sig)
-          end
+      def print(*args)
+        unless config.quiet
+          stdout.puts args.join(' ')
         end
       end
 
-      def main
-        Thread.new do
-          info 'watching for outsized Sidekiq processes...'
-          loop do
-            sleep 10
-            check_pids
-            break unless @watch_children
+      def stop!(code = 0)
+        if worker_pool
+          unless worker_pool.stopped?
+            worker_pool.stop!
+            sleep 5
           end
-          info 'leaving the main loop..'
         end
-      end
 
-      def info(*args)
-        @logger.info(*args)
+        kernel.exit(code)
       end
 
-      def error(*args, exception: nil)
-        @logger.error("exception: #{exception.message}") if exception
-        @logger.error(*args)
-      end
-
-      def handle_sig(sig)
-        print "received OS signal #{sig}"
-        # If we're shutting down, we don't need to re-spawn child processes that die
-        @watch_children = false if sig == 'INT' || sig == 'TERM'
-        @pids.each do |pid|
-          Process.kill(sig, pid)
+      def close_logger
+        self.logger.close if logger && logger.respond_to?(:close)
+        if Process.pid != master_pid
+          log_device.close if log_device && log_device.respond_to?(:close)
         end
+      rescue
+        nil
       end
 
-      def fork_child(index)
-        require 'sidekiq'
-        require 'sidekiq/cli'
-
-        Process.fork do
-          process_argv = sidekiq_argv.dup << '-P' << pid_file(index)
-          process_argv << '--tag' << "sidekiq.#{name}.#{index + 1}"
-          info "starting up sidekiq instance #{index} with ARGV: 'bundle exec sidekiq #{process_argv.join(' ')}'"
-          begin
-            cli = Sidekiq::CLI.instance
-            cli.parse process_argv
-            cli.run
-          rescue => e
-            raise e if $DEBUG
-            error e.message
-            error e.backtrace.join("\n")
-            stop!(1)
-          end
+      def init_logger
+        if config.logfile
+          ::FileUtils.mkdir_p(::File.dirname(config.logfile))
+          self.log_device = ::File.open(config.logfile, 'a', 0644)
+        else
+          self.log_device = stdout
         end
-      end
 
-      def pid_file(index)
-        "#{pid_prefix}.#{index + 1}"
-      end
+        log_device.sync = true
 
-      def start_children
-        Array.new(process_count) do |index|
-          pid = fork_child(index)
-          ProcessDescriptor.new(pid, index, pid_file(index))
-        end
+        @logger = ::Logger.new(log_device, level: ::Logger::INFO)
+        info('opening log file: ' + (config.logfile ? config.logfile.to_s : 'STDOUT'))
       end
 
-      def check_pids
-        print_info         = false
-        @last_info_printed ||= Time.now.to_i
-        if Time.now.to_i - @last_info_printed > 60
-          @last_info_printed = Time.now.to_i
-          print_info         = true
-        end
-
-        pids.each do |pid|
-          memory_percent_used = `ps -o %mem= -p #{pid}`.to_f
-          info "sidekiq.#{name % '%15s'}[#{processes[pid].index}] —— pid=#{pid.to_s % '%6d'} —— memory pct=#{memory_percent_used.round(2)}%" if print_info
-          if memory_percent_used == 0.0 # child died
-            restart_dead_child(pid)
-          elsif memory_percent_used > per_process_memory_limit
-            info "pid #{pid} crossed memory threshold, used #{memory_percent_used.round(2)}% of RAM, exceeded #{per_process_memory_limit}% —> replacing..."
-            restart_oversized_child(pid)
-          elsif $DEBUG
-            info "#{pid}: #{memory_percent_used.round(2)}"
-          end
-        end
+      def print_header
+        print "Sidekiq Cluster v#{Sidekiq::Cluster::VERSION} — Starting up, Cluster name: [#{config.name}]"
+        print "  • max memory limit is  : #{config.max_memory_percent.round(2)}%"
+        print "  • memory strategy      : #{config.memory_strategy.to_s.capitalize}"
+        print "  • number of workers is : #{config.process_count}"
+        print "  • logfile              : #{config.logfile}" if config.logfile
+        print "  • pid file path        : #{config.pid_prefix}" if config.pid_prefix
       end
 
-      def restart_oversized_child(pid)
-        @pids.delete(pid)
-        Process.kill('USR1', pid)
-        sleep 5
-        Process.kill('TERM', pid)
-        @pids << fork_child
+      def prefix
+        Process.pid == master_pid ? ' «master» '.bold.blue : ' «worker» '.bold.green
       end
 
-      def replace_pid(old_pid, new_pid)
-        pd       = processes[old_pid]
-        pid_file = pid_file(pd.index)
-
-        ::File.unlink(pid_file) if ::File.exist?(pidfile)
-        pids.delete(old_pid)
-        processes.delete(old_pid)
-
-        pd.pid = new_pid
-        processes[new_pid] = pd
+      def info(*args)
+        logger.info(prefix + args.join(' '))
       end
 
-      def restart_dead_child(pid)
-        info "pid=#{pid} died, restarting..."
-
-        pd = processes[pid]
-        raise ArgumentError, "Unregistered pid found #{pid}, no existing descriptor found!" unless pd
-        new_pid = fork_child(pd.index)
-        replace_pid(pid, new_pid)
-
-        info "replaced lost pid #{pid} with #{new_pid}"
+      def error(*args, exception: nil)
+        logger.error(prefix + "exception: #{exception.message}") if exception
+        logger.error(prefix + args.join(' '))
       end
 
-      def parse_args!
-        options = {}
-        banner  = "USAGE".bold.blue + "\n     sidekiq-cluster [options] -- [sidekiq-options]".bold.green
-        parser  = ::OptionParser.new(banner) do |opts|
-          opts.separator ''
-          opts.separator 'EXAMPLES'.bold.blue
-          opts.separator '    $ cd rails_app'.bold.magenta
-          opts.separator '    $ bundle exec sidekiq-cluster -N 2 -- -c 10 -q default,12 -l log/sidekiq.log'.bold.magenta
-          opts.separator ' '
-          opts.separator 'SIDEKIQ CLUSTER OPTIONS'.bold.blue
-
-          opts.on('-n', '--name=NAME',
-                  'the name of this cluster, used when ',
-                  'when running multiple clusters', ' ') do |v|
-            self.name = v
-          end
-          opts.on('-P', '--pidfile=FILE',
-                  'Pidfile prefix, ',
-                  'eg "/var/www/shared/config/sidekiq.pid"', ' ') do |v|
-            self.pid_prefix = v
+      private
+
+      def parse_cli_arguments
+        banner = 'USAGE'.bold.yellow + "\n    sidekiq-cluster [options] -- ".bold.magenta + "[sidekiq-options]".bold.cyan
+        parser = ::OptionParser.new(banner, 26, indent = ' ' * 3) do |opts|
+          opts.separator ::Sidekiq::Cluster::BANNER
+          opts.on('-n', '--name NAME',
+                  'the name of this cluster, used when running ',
+                  'multiple clusters.', ' ') do |v|
+            config.name = v
           end
-          opts.on('-L', '--logfile=FILE',
-                  'Logfile for the cluster script', ' ') do |v|
-            self.log_device = v
-            init_logger!
+          opts.on('-N', '--num-processes NUM',
+                  'Number of worker processes to use for this cluster,',
+                  'defaults to the number of cores - 1', ' ') do |v|
+            config.process_count = v.to_i
           end
-          opts.on('-M', '--max-memory=PERCENT',
-                  'Maximum percent RAM that this',
+
+          opts.on('-M', '--max-memory PCT',
+                  'Float, the maximum percent total RAM that this',
                   'cluster should not exceed. Defaults to ' +
                       Sidekiq::Cluster::MAX_RAM_PCT.to_s + '%.', ' ') do |v|
-            self.memory_percentage_limit = v.to_f
+            config.max_memory_percent = v.to_f
+          end
+
+          opts.on('-m', '--memory-mode MODE',
+                  'Either "total" (default) or "individual".', ' ',
+                  '• In the "total" mode the largest worker is restarted if ',
+                  '  the sum of all workers exceeds the memory limit. ',
+                  '• In the "individual" mode, a worker is restarted if it ',
+                  '  exceeds MaxMemory / NumWorkers.', ' ') do |v|
+            if Config::MEMORY_STRATEGIES.include?(v.downcase.to_sym)
+              config.memory_strategy = v.downcase.to_sym
+            else
+              raise "Invalid strategy '#{v}'. Valid strategies are :#{Config::MEMORY_STRATEGIES.join(', ')}"
+            end
+          end
+
+          opts.on('-P', '--pid-prefix PATH',
+                  'Pidfile prefix path used to generate pid files, ',
+                  'eg "/var/tmp/cluster.pid"', ' ') do |v|
+            config.pid_prefix = v
           end
-          opts.on('-N', '--num-processes=NUM',
-                  'Number of processes to start,',
-                  'defaults to number of cores - 1', ' ') do |v|
-            self.process_count = v.to_i
+
+          opts.on('-L', '--logfile FILE',
+                  'The logfile for sidekiq cluster itself', ' ') do |v|
+            config.logfile = v
+          end
+
+          opts.on('-w', '--work-dir DIR',
+                  'Directory where to run', ' ') do |v|
+            config.work_dir = v
           end
+
           opts.on('-q', '--quiet',
-                  'Do not log to STDOUT') do |v|
-            self.logger = Logger.new(nil)
+                  'Do not print anything to STDOUT') do |_v|
+            config.quiet = true
           end
+
           opts.on('-d', '--debug',
-                  'Print debugging info before starting sidekiqs') do |_v|
-            options[:debug] = true
+                  'Print debugging info before starting workers') do |_v|
+            config.debug = true
           end
+
           opts.on('-h', '--help', 'this help') do |_v|
-            self.print opts
+            config.help = true
+            stdout.puts opts
             stop!
           end
         end
-        parser.order!(argv)
-        print("debug: #{self.inspect}") if options[:debug]
+
+        parser.order!(argv.dup)
       end
     end
   end

data/lib/sidekiq/cluster/config.rb

@@ -0,0 +1,53 @@
+require 'etc'
+require 'dry-configurable'
+require_relative 'version'
+
+module Sidekiq
+  module Cluster
+    class Config
+      MEMORY_STRATEGIES = %i(total individual)
+      ARGV_SEPARATOR = '--'.freeze
+
+      extend ::Dry::Configurable
+
+      setting :spawn_block, ->(worker) {
+        require 'sidekiq'
+        require 'sidekiq/cli'
+
+        process_argv = worker.config.worker_argv.dup << '-P' << worker.pid_file
+        process_argv << '--tag' << "sidekiq.#{worker.config.name}.#{worker.index + 1}"
+
+        cli = Sidekiq::CLI.instance
+        cli.parse %w(bundle exec sidekiq) + process_argv
+        cli.run
+      }
+
+      setting :name, 'default'
+      setting :pid_prefix, '/var/tmp/sidekiq-cluster.pid'
+      setting :process_count, Etc.nprocessors - 1
+      setting :max_memory_percent, MAX_RAM_PCT
+      setting :memory_strategy, :total # also supported :individual
+      setting :logfile
+      setting :work_dir, Dir.pwd
+      setting :cluster_argv, []
+      setting :worker_argv, []
+      setting :debug, false
+      setting :quiet, false
+      setting :help, false
+
+
+      def self.split_argv(argv)
+        configure do |c|
+          if argv.index(ARGV_SEPARATOR)
+            c.worker_argv  = argv[(argv.index(ARGV_SEPARATOR) + 1)..-1] || []
+            c.cluster_argv = argv[0..(argv.index(ARGV_SEPARATOR) - 1)] || []
+          else
+            c.worker_argv = []
+            c.cluster_argv = argv
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/sidekiq/cluster/memory.rb

@@ -0,0 +1,25 @@
+module Sidekiq
+  module Cluster
+    module Memory
+      class << self
+        attr_accessor :strategies
+
+        def offenders(worker_pool)
+          name = worker_pool.config.memory_strategy.to_sym
+          strategies[name].new(worker_pool).offenders
+        end
+      end
+
+      self.strategies ||= {}
+
+      module MemoryStrategy
+        def self.included(base)
+          ::Sidekiq::Cluster::Memory.strategies[base.name.gsub(/.*::/, '').downcase.to_sym] = base
+        end
+      end
+    end
+  end
+end
+
+require_relative 'memory/individual'
+require_relative 'memory/total'

data/lib/sidekiq/cluster/memory/individual.rb

@@ -0,0 +1,23 @@
+module Sidekiq
+  module Cluster
+    module Memory
+      class Individual
+        include MemoryStrategy
+
+        attr_accessor :config, :worker_pool
+
+        def initialize(worker_pool)
+          self.worker_pool = worker_pool
+          self.config      = worker_pool.config
+        end
+
+        def offenders
+          worker_pool.find do |worker|
+            worker.memory_used_pct > config.max_memory_percent / worker_pool.size
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/sidekiq/cluster/memory/total.rb

@@ -0,0 +1,20 @@
+require_relative '../stdlib_ext'
+require_relative 'individual'
+module Sidekiq
+  module Cluster
+    module Memory
+      class Total < Individual
+        include MemoryStrategy
+
+        def offenders
+          total_ram_pct = worker_pool.map(&:memory_used_pct).sum
+          worker_pool.cli.info("total RAM used by workers is #{'%.2f%%' % total_ram_pct}")
+          if total_ram_pct > config.max_memory_percent
+            worker_pool.sort_by(&:memory_used_pct).inverse[0..1]
+          end || []
+        end
+      end
+    end
+  end
+end
+

data/lib/sidekiq/cluster/monitors/base.rb

@@ -0,0 +1,42 @@
+module Sidekiq
+  module Cluster
+    module Monitors
+      SLEEP_DELAY = 5
+      LOGGING_PERIOD = 30
+
+      class Base
+        attr_accessor :pool, :thread
+
+        def initialize(pool)
+          self.pool = pool
+          @last_logged_at = Time.now.to_i
+        end
+
+        def start
+          self.thread = Thread.new { self.monitor }
+          self
+        end
+
+        def join
+          thread.join if thread
+        end
+
+        def monitor
+          raise 'Abstract method'
+        end
+
+        def log_periodically(msg, &block)
+          t = Time.now.to_i
+          if t - @last_logged_at > LOGGING_PERIOD
+            pool.cli.info(msg) if msg
+            Array(block.call).each do |result|
+              pool.cli.info(result)
+            end if block
+            @last_logged_at = t
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/sidekiq/cluster/monitors/dead_children.rb

@@ -0,0 +1,21 @@
+require_relative 'base'
+
+module Sidekiq
+  module Cluster
+    module Monitors
+      class DeadChildren < Base
+        def monitor
+          pool.info 'watching for workers that died...'
+          loop do
+            sleep SLEEP_DELAY - 1
+            pool.workers.each(&:check_worker)
+            break unless pool.operational?
+            "monitor for Dead Children is operational, last logged at #{@last_logged_at}"
+          end
+          pool.info 'leaving Dead Children Monitor'
+        end
+      end
+    end
+  end
+end
+

data/lib/sidekiq/cluster/monitors/oom.rb

@@ -0,0 +1,22 @@
+require_relative 'dead_children'
+require_relative '../memory'
+module Sidekiq
+  module Cluster
+    module Monitors
+      class OOM < Base
+        def monitor
+          pool.info 'watching for worker processes exceeding size threshold'
+          loop do
+            sleep SLEEP_DELAY + 1
+            ::Sidekiq::Cluster::Memory.offenders(pool).each { |worker| worker.respawn! }
+            break unless pool.operational?
+            log_periodically "monitor for Out Of Memory is operational, last logged at #{@last_logged_at}" do
+              pool.workers.map(&:status)
+            end
+          end
+          pool.info 'leaving Memory Monitor.'
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/cluster/stdlib_ext.rb

@@ -0,0 +1,5 @@
+class Array
+  def sum
+    reduce(:+)
+  end
+end

data/lib/sidekiq/cluster/version.rb

@@ -1,6 +1,35 @@
+# frozen_string_literal: true
+
+require 'colored2'
+
 module Sidekiq
   module Cluster
-    VERSION = '0.1.1'
+    VERSION ||= '0.1.2'
+
+    DESCRIPTION = <<-eof
+      This library provides CLI interface for starting multiple copies of Sidekiq in parallel, 
+      typically to take advantage of multi-core systems.  By default it starts N - 1 processes, 
+      where N is the number of cores on the current system. Sidekiq Cluster is controlled with CLI 
+      flags that appear before `--` (double dash), while any arguments that follow double dash are 
+      passed to each Sidekiq process. The exception is the `-P pidfile`, which clustering script 
+      passes to each sidekiq process individually.
+    eof
+                      .gsub(/\n    /, ' ').freeze
+
+    MAX_RAM_PCT ||= 80.freeze
+
+    # @formatter:off
+      BANNER ||= %Q(
+#{'EXAMPLES'.bold.yellow}
+
+    $ cd rails_app
+    $ echo 'gem "sidekiq-cluster"' >> Gemfile
+    $ bundle install
+    #{'$ bundle exec sidekiq-cluster -N 2 '.bold.magenta + '--' + ' -c 10 -q default,12 -L log/sidekiq.log'.bold.cyan}
+
+#{'OPTIONS'.bold.yellow}
+).freeze
+      # @formatter:on
   end
 end
 

data/lib/sidekiq/cluster/worker.rb

@@ -0,0 +1,114 @@
+require 'forwardable'
+require 'state_machines'
+
+module Sidekiq
+  module Cluster
+    class Worker
+
+      state_machine :state, initial: :idle do
+        event :start do
+          transition [:idle] => :starting
+        end
+
+        event :started do
+          transition [:starting] => :running
+        end
+
+        event :stop do
+          transition [:idle, :starting, :running] => :stopping
+        end
+
+        event :shutdown do
+          transition [:idle, :starting, :running, :stopping] => :stopped
+        end
+      end
+
+      extend Forwardable
+      def_delegators :@cli, :info, :error, :print, :stdout, :stdin, :stderr, :kernel, :master_pid
+
+      attr_accessor :pid, :index, :cli, :config, :state
+
+      def initialize(index, cli)
+        self.config = cli.config
+        self.index  = index
+        self.cli    = cli
+        self.state  = :idle
+      end
+
+      def spawn
+        if master?
+          cli.info "booting worker #{'%2d' % index} with ARGV '#{config.worker_argv.join(' ')}'"
+          self.pid = ::Process.fork do
+            # cli.close_logger
+            cli.init_logger
+
+            cli.info "child #{index}, running spawn block..."
+            begin
+              config.spawn_block[self]
+            rescue => e
+              raise e if $DEBUG
+              error e.message
+              error e.backtrace.join("\n")
+              stop!
+            end
+          end
+        end
+        pid
+      end
+
+      def memory_used_pct
+        self.pid = Process.pid if pid.nil?
+        result = `ps -o %mem= -p #{pid}`
+        result.nil? || result == '' ? -1 : result.to_f
+      end
+
+      def memory_used_percent
+        mem = memory_used_pct
+        mem < 0 ? 'DEAD' : sprintf('%.2f%', mem)
+      end
+
+      def check_worker
+        self.pid = Process.pid if pid.nil?
+        respawn! if memory_used_pct == -1 && master?
+      end
+
+      def respawn!
+        if master? && pid
+          cli.info "NOTE: re-spawning child #{index} (pid #{pid}), memory is at #{memory_used_percent}."
+          begin
+            Process.kill('USR1', pid)
+            sleep 5
+            Process.kill('TERM', pid)
+          rescue Errno::ESRCH
+            nil
+          end
+          self.pid = nil
+          spawn
+        end
+      end
+
+      def handle_signal(sig)
+        Process.kill(sig, pid) if pid
+      rescue Errno::ESRCH
+        nil
+      end
+
+      def pid_file
+        "#{config.pid_prefix}.#{index + 1}"
+      end
+
+      def master?
+        Process.pid == cli.master_pid
+      end
+
+      def log_worker
+        cli.info(status) if memory_used_pct >= 0.0
+      end
+
+      def status
+        "worker.#{config.name}[index=#{index}, pid=#{pid ? pid : 'nil'}] —— memory: #{memory_used_percent}"
+      end
+    end
+  end
+end
+

data/lib/sidekiq/cluster/worker_pool.rb

@@ -0,0 +1,113 @@
+require 'forwardable'
+require 'state_machines'
+
+require_relative 'worker'
+require_relative 'memory'
+
+require_relative 'monitors/oom'
+require_relative 'monitors/dead_children'
+
+module Sidekiq
+  module Cluster
+    class WorkerPool
+
+      include Enumerable
+
+      state_machine :state, initial: :idle do
+        event :start do
+          transition [:idle] => :starting
+        end
+
+        event :started do
+          transition [:starting] => :running
+        end
+
+        event :stop do
+          transition [:idle, :starting, :running, :stopping] => :stopping
+        end
+
+        event :shutdown do
+          transition [:idle, :starting, :running, :stopping] => :stopped
+        end
+
+        after_transition any => :stopped do |pool, _transition|
+          pool.cli.stop!
+        end
+
+        state :idle, :starting, :started, :stopping, :stopped
+      end
+
+      extend Forwardable
+      def_delegators :@cli, :info, :error, :print, :stdout, :stdin, :stderr, :kernel
+      def_delegators :@workers, :each
+
+      attr_accessor :workers, :config, :cli, :process_count, :monitors
+
+      MONITORS = [Monitors::DeadChildren, Monitors::OOM]
+
+      def initialize(cli, config)
+        self.cli           = cli
+        self.config        = config
+        self.process_count = config.process_count
+        self.workers       = []
+        self.monitors      = []
+        self.state         = :idle
+
+        @signal_received = false
+      end
+
+      def spawn
+        start!
+
+        create_workers
+
+        info "spawning #{workers.size} workers..."
+        workers.each(&:spawn)
+
+        info "worker pids: #{workers.map(&:pid)}"
+
+        setup_signal_traps
+        started!
+
+        start_monitors
+
+        info 'startup successful'
+
+        Process.waitall
+        monitors.each(&:join)
+
+        info 'all children exited, shutting down'
+      end
+
+      def operational?
+        shutdown! if stopping?
+        stop! if @signal_received
+        state_name == :running
+      end
+
+      private
+
+      def create_workers
+        process_count.times { |index| self.workers << Worker.new(index, cli) }
+      end
+
+      def start_monitors
+        self.monitors = MONITORS.map { |monitor| monitor.new(self).start }
+      end
+
+      def setup_signal_traps
+        %w(INT USR1 TERM).each do |sig|
+          Signal.trap(sig) do
+            handle_signal(sig)
+          end
+        end
+      end
+
+      def handle_signal(sig)
+        cli.stderr.puts "received OS signal #{sig}"
+        @signal_received = true if (sig == 'INT' || sig == 'TERM' || sig == 'STOP')
+        workers.each { |w| w.handle_signal(sig) }
+      end
+    end
+  end
+end

data/sidekiq-cluster.gemspec

@@ -4,10 +4,6 @@ lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'sidekiq/cluster/version'
 
-Sidekiq::Cluster::DESCRIPTION = <<-eof
-  This library provides CLI interface for starting multiple copies of Sidekiq in parallel, typically to take advantage of multi-core systems.  By default it starts N - 1 processes, where N is the number of cores on the current system. Sidekiq Cluster is controlled with CLI flags that appear before `--` (double dash), while any arguments that follow double dash are passed to each Sidekiq process. The exception is the `-P pidfile`, which clustering script passes to each sidekiq process individually.
-eof
-
 Gem::Specification.new do |spec|
   spec.name          = 'sidekiq-cluster'
   spec.version       = Sidekiq::Cluster::VERSION
@@ -28,6 +24,8 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'colored2' 
   spec.add_dependency 'sidekiq' 
+  spec.add_dependency 'dry-configurable'
+  spec.add_dependency 'state_machines'
 
   spec.add_development_dependency 'rspec', '~> 3.5.0'
   spec.add_development_dependency 'rspec-its'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-cluster
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Konstantin Gredeskoul
@@ -38,6 +38,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: dry-configurable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: state_machines
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -123,7 +151,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: |2
-    This library provides CLI interface for starting multiple copies of Sidekiq in parallel, typically to take advantage of multi-core systems.  By default it starts N - 1 processes, where N is the number of cores on the current system. Sidekiq Cluster is controlled with CLI flags that appear before `--` (double dash), while any arguments that follow double dash are passed to each Sidekiq process. The exception is the `-P pidfile`, which clustering script passes to each sidekiq process individually.
+        This library provides CLI interface for starting multiple copies of Sidekiq in parallel,    typically to take advantage of multi-core systems.  By default it starts N - 1 processes,    where N is the number of cores on the current system. Sidekiq Cluster is controlled with CLI    flags that appear before `--` (double dash), while any arguments that follow double dash are    passed to each Sidekiq process. The exception is the `-P pidfile`, which clustering script    passes to each sidekiq process individually.
 email:
 - kigster@gmail.com
 executables:
@@ -142,7 +170,17 @@ files:
 - exe/sidekiq-cluster
 - lib/sidekiq/cluster.rb
 - lib/sidekiq/cluster/cli.rb
+- lib/sidekiq/cluster/config.rb
+- lib/sidekiq/cluster/memory.rb
+- lib/sidekiq/cluster/memory/individual.rb
+- lib/sidekiq/cluster/memory/total.rb
+- lib/sidekiq/cluster/monitors/base.rb
+- lib/sidekiq/cluster/monitors/dead_children.rb
+- lib/sidekiq/cluster/monitors/oom.rb
+- lib/sidekiq/cluster/stdlib_ext.rb
 - lib/sidekiq/cluster/version.rb
+- lib/sidekiq/cluster/worker.rb
+- lib/sidekiq/cluster/worker_pool.rb
 - sidekiq-cluster.gemspec
 homepage: https://github.com/kigster/sidekiq-cluster
 licenses:
@@ -168,9 +206,10 @@ rubygems_version: 2.4.5.2
 signing_key: 
 specification_version: 4
 summary: This library provides CLI interface for starting multiple copies of Sidekiq
-  in parallel, typically to take advantage of multi-core systems.  By default it starts
-  N - 1 processes, where N is the number of cores on the current system. Sidekiq Cluster
-  is controlled with CLI flags that appear before `--` (double dash), while any arguments
-  that follow double dash are passed to each Sidekiq process. The exception is the
-  `-P pidfile`, which clustering script passes to each sidekiq process individually.
+  in parallel,    typically to take advantage of multi-core systems.  By default it
+  starts N - 1 processes,    where N is the number of cores on the current system.
+  Sidekiq Cluster is controlled with CLI    flags that appear before `--` (double
+  dash), while any arguments that follow double dash are    passed to each Sidekiq
+  process. The exception is the `-P pidfile`, which clustering script    passes to
+  each sidekiq process individually.
 test_files: []