sidekiq-process_manager 1.1.1 → 1.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 51d57e09aba69d64225948647f2f95625c668b44eb16e514b5ab4daf8e064e3c
-  data.tar.gz: 73d49c20ce5bf06a8d84a168d8e1afa2b04883ea1656d3d4e0ac96b66855140c
+  metadata.gz: dc25ffae388e5ca7ad73f0976d00cf09ec84124b2284a000af1dc0970cf87c20
+  data.tar.gz: 94637ae3cfa8a6fa991e95ee434c8410936c62263695f36b6db8c2b664d1f439
 SHA512:
-  metadata.gz: 2f35e8a6ccbbcbaa15fbbbd62658bd79e4337e675a0c4a776e8dc2c6457bb957ed5ba9880336532a68f54e5ea13890917d758d5f84a2395130c0c52a5166a7cc
-  data.tar.gz: dd4fa55ec09584a890e5125d93a64d15924b138583d23e91331a1818bec0c58dfa0cdd81ab58bfab37951a645ed41e149fa04996ec89d11c85c0362204c21be4
+  metadata.gz: 32bba980929441cb06022c829f73e7a7d8ee41de0dba07c98f44cb1f2c339dab87ec21dae8b1847ae2967e5771fb6506dd09a42a9540d784df4defd9f820f664
+  data.tar.gz: e1dc4e4e0e502b0747932b5b12011acfcb04535baeaae835027c50e1ec3ae1ff6ea7c0f6317469a8c4e11f8d9d9bfe40c530064ad4da5b4354231af1182776c6

data/CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 1.1.2
+
+### Added
+- Added thread to monitor child processes to make sure they exit after a SIGTERM signal has been sent. If a process does not exit after the configured Sidekiq timeout time, then it will be killed with a SIGKILL signal.
+
+### Changed
+- A SIGINT sent to the manager process will sent SIGTERM to the child processes to give them a chance to shutdown gracefully.
+
 ## 1.1.1
 
 ### Added

data/README.md

@@ -8,7 +8,7 @@ This gem provides a command line script for managing [sidekiq](https://github.co
 
 The sidekiq processes can all be managed by sending signals to the manager process. This process simply forwards the signals on to the child processes, allowing you to control the sidekiq processes as you normally would.
 
-If one of the sidekiq processes dies unexpectedly, the process manager automatically starts a new sidekiq process to replace it.
+If one of the sidekiq processes exits unexpectedly, the process manager automatically starts a new sidekiq process to replace it.
 
 ## Pre-Forking
 
@@ -44,7 +44,7 @@ For a Rails application, you would normally want to preboot the `config/boot.rb`
 
 ## Memory Bloat
 
-You can also specify a maximum memory footprint that you want to allow for each child process. You can use this feature to automatically guard against poorly designed workers that bloat the Ruby memory heap. Note that you can also use an external process monitor to kill processes with memory bloat; the process manager will restart any process regardless of how it dies.
+You can also specify a maximum memory footprint that you want to allow for each child process. You can use this feature to automatically guard against poorly designed workers that bloat the Ruby memory heap. Note that you can also use an external process monitor to kill processes with memory bloat; the process manager will restart any process regardless of how it exits.
 
 ## Usage
 

data/VERSION

@@ -1 +1 @@
-1.1.1
+1.1.2

data/lib/sidekiq/process_manager/manager.rb

@@ -72,7 +72,7 @@ module Sidekiq
               signal = @signal_pipe_read.gets.strip
               send_signal_to_children(signal.to_sym)
             rescue => e
-              log_error("Error handling signal #{signal}: #{e.message}")
+              log_warning("Error sending signal #{signal} to child processes: #{e.message}")
             end
           end
         end
@@ -81,6 +81,7 @@ module Sidekiq
         [:INT, :TERM, :USR1, :USR2, :TSTP, :TTIN].each do |signal|
           ::Signal.trap(signal) do
             if ::Process.pid == master_pid
+              signal = :TERM if signal == :INT
               @signal_pipe_write.puts(signal)
             end
           end
@@ -88,8 +89,12 @@ module Sidekiq
 
         # Ensure that child processes receive the term signal when the master process exits.
         at_exit do
-          if ::Process.pid == master_pid && @process_count > 0
-            send_signal_to_children(:TERM)
+          if ::Process.pid == master_pid
+            if @process_count > 0
+              send_signal_to_children(:TERM)
+            end
+            wait_for_children_to_exit
+            log_info("Process manager exiting")
           end
         end
 
@@ -104,9 +109,8 @@ module Sidekiq
 
         start_memory_monitor if @max_memory
 
-        log_info("Process manager started with pid #{::Process.pid}")
+        log_info("Process manager started")
         monitor_child_processes
-        log_info("Process manager #{::Process.pid} exiting")
       end
 
       # Helper to wait on the manager to wait on child processes to start up.
@@ -237,14 +241,24 @@ module Sidekiq
         log_info("Process manager trapped signal #{signal}")
         @process_count = 0 if signal == :INT || signal == :TERM
         pids.each do |pid|
-          begin
-            log_info("Sending signal #{signal} to sidekiq process #{pid}")
-            ::Process.kill(signal, pid)
-          rescue => e
-            log_warning("Error sending signal #{signal} to sidekiq process #{pid}: #{e.inspect}")
+          send_signal_to_pid(signal, pid)
+        end
+      end
+
+      def send_signal_to_pid(signal, pid)
+        signal = signal.to_sym
+        begin
+          log_info("Sending signal #{signal} to sidekiq process #{pid}")
+          ::Process.kill(signal, pid)
+          if [:TERM, :INT].include?(signal)
+            Thread.new do
+              Thread.current.name = "pid-#{pid}-killer"
+              ensure_pid_dies(pid)
+            end
           end
+        rescue Errno::ESRCH
+          # The process is already dead
         end
-        wait_for_children_to_exit(pids) if @process_count == 0
       end
 
       def start_memory_monitor
@@ -259,12 +273,8 @@ module Sidekiq
               begin
                 memory = GetProcessMem.new(pid)
                 if memory.bytes > @max_memory
-                  log_warning("Kill bloated sidekiq process #{pid}: #{memory.mb.round}mb used")
-                  begin
-                    ::Process.kill(:TERM, pid)
-                  rescue Errno::ESRCH
-                    # The process is already dead
-                  end
+                  log_warning("Killing bloated sidekiq process #{pid}: #{memory.mb.round}mb used")
+                  send_signal_to_pid(:TERM, pid)
                   break
                 end
               rescue => e
@@ -281,22 +291,13 @@ module Sidekiq
         end
       end
 
-      def wait_for_children_to_exit(pids)
-        timeout = monotonic_time + (sidekiq_options[:timeout] || 25).to_f
+      def wait_for_children_to_exit
         pids.each do |pid|
-          while monotonic_time < timeout
-            break unless process_alive?(pid)
+          while process_alive?(pid)
             sleep(0.01)
           end
         end
-
-        pids.each do |pid|
-          begin
-            ::Process.kill(:INT, pid) if process_alive?(pid)
-          rescue
-            # Ignore errors so we can continue to kill other processes.
-          end
-        end
+        log_info("All sidekiq processes have exited")
       end
 
       def process_alive?(pid)
@@ -308,6 +309,24 @@ module Sidekiq
         end
       end
 
+      def ensure_pid_dies(pid)
+        # Wait for the process to die, or kill it after a timeout.
+        timeout = (sidekiq_options[:timeout] || 25).to_f
+        end_time = monotonic_time + timeout
+        while monotonic_time < end_time && process_alive?(pid)
+          sleep(0.1)
+        end
+
+        if process_alive?(pid)
+          begin
+            ::Process.kill(:KILL, pid)
+            log_warning("Sidekiq process #{pid} failed to exit after #{timeout} seconds; killed with SIGKILL")
+          rescue Errno::ESRCH
+            # The process is already dead
+          end
+        end
+      end
+
       # Listen for child processes dying and restart if necessary.
       def monitor_child_processes
         loop do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-process_manager
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.1.2
 platform: ruby
 authors:
 - Brian Durand
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-07-19 00:00:00.000000000 Z
+date: 2023-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq