sidekiq-mem-warden 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 32004d7f9bc76aa4ed76cd96fe7f23bd1a4159fc51aad23254d894ced12a01d2
-  data.tar.gz: a2cddf656f6f022041e084e58cbe598b71b169a3bd2c8d3e4a400f8f7efdb2cf
+  metadata.gz: e1a5f747821df9e7e8d6f2b15a195b11ef025b2f37210fb1c4a7e43ad88757e9
+  data.tar.gz: 42fb14442d1d7b9f9c74950f539c8e3ca2dc844657c2246a9940cb2e18498d6f
 SHA512:
-  metadata.gz: f2813c65ffc5a0279e24a8780abb00942c3c932aaff25459141b559b887b272bc747d3fe9abe9b087a2ad2a001702fbb89aa7e8a99c02d40df80d0cb60039516
-  data.tar.gz: 4d91e86d78fdb0d4b87ebf7f226464443c1c144361526516fb886838e8beb01064e276fa246d3019555ca52f26ccd3146517a00c1c2bc9612099c767caab1620
+  metadata.gz: 7fbf5b2e812b1be8a361b74d6417111a3cf05dc60cf549ee1d73c7d67a7c8f535f689804f93204fa05d57d6af4df5c697f891ec782a2d542df648f3677b79f3b
+  data.tar.gz: 2b861f4c40a464a6c81a473f02762143d40538117dcc98b56ec31b835b17b715a9df9b0d33305cd4bd40e2d4d9b63ec0ef23eb393e07267a82956cad3dbebff2

data/CHANGELOG.md

@@ -8,4 +8,7 @@
 
 ## 0.1.2
 - Default memory limit set to 1 GB (1024 MB).
-- Default quiet timeout set to 5 minutes (300 seconds).
+- Default grace time set to 5 minutes (300 seconds).
+
+## 0.1.3
+- Replaced the internal watchdog with Sidekiq server middleware based on sidekiq-worker-killer.

data/Gemfile.lock

@@ -1,17 +1,25 @@
 PATH
   remote: .
   specs:
-    sidekiq-mem-warden (0.1.2)
-      sidekiq (>= 6.0)
+    sidekiq-mem-warden (0.1.3)
+      get_process_mem (>= 0.2.0)
+      sidekiq (>= 7.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
+    bigdecimal (3.3.1)
+    concurrent-ruby (1.3.6)
     connection_pool (2.5.3)
     diff-lcs (1.6.2)
+    ffi (1.17.3-arm64-darwin)
+    get_process_mem (1.0.0)
+      bigdecimal (>= 2.0)
+      ffi (~> 1.0)
     rack (2.2.17)
     rake (13.3.0)
-    redis (4.8.1)
+    redis-client (0.26.2)
+      connection_pool
     rspec (3.13.1)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
@@ -25,14 +33,14 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.6)
-    sidekiq (6.5.12)
-      connection_pool (>= 2.2.5, < 3)
-      rack (~> 2.0)
-      redis (>= 4.5.0, < 5)
+    sidekiq (7.1.6)
+      concurrent-ruby (< 2)
+      connection_pool (>= 2.3.0)
+      rack (>= 2.2.4)
+      redis-client (>= 0.14.0)
 
 PLATFORMS
   arm64-darwin-24
-  ruby
 
 DEPENDENCIES
   bundler (>= 1.17)

data/README.md

@@ -1,6 +1,6 @@
 # Sidekiq::Mem::Warden
 
-Sidekiq::Mem::Warden is a tiny, in-process watchdog that quiets a Sidekiq process when RSS exceeds a limit, waits for busy jobs to drain, then exits so your process manager can restart it.
+Sidekiq::Mem::Warden is a tiny Sidekiq server middleware that watches RSS and shuts down bloated processes after a grace period, allowing your process manager to restart them.
 
 ## Installation
 
@@ -26,21 +26,23 @@ Configure it in your Sidekiq server process:
 require "sidekiq/mem/warden"
 
 Sidekiq.configure_server do |config|
-  Sidekiq::Mem::Warden.configure do |c|
-    c.memory_limit_mb = 1024
-    c.check_interval = 15
-    c.quiet_timeout = 30
-    c.shutdown_timeout = 300
+  config.server_middleware do |chain|
+    chain.add Sidekiq::Mem::Warden,
+      max_rss: 1024,
+      grace_time: 300,
+      shutdown_wait: 30,
+      kill_signal: "SIGKILL",
+      gc: true,
+      skip_shutdown_if: ->(worker, job, queue) { false },
+      on_shutdown: ->(worker, job, queue) { nil }
   end
-
-  Sidekiq::Mem::Warden.install!(config)
 end
 ```
 
 Operational notes:
 
 - The warden runs inside each Sidekiq process.
-- When memory exceeds the limit, it sends `TSTP` to quiet the process, sleeps for `quiet_timeout`, waits for `Sidekiq::Workers` to drain, then sends `TERM` to exit.
+- When RSS exceeds `max_rss`, it quiets the process, waits for jobs to finish up to `grace_time`, then stops and sends `kill_signal`.
 - Ensure your supervisor (systemd, Kubernetes, etc.) is set to restart the process.
 
 ## Development
@@ -53,6 +55,10 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/tedaford/sidekiq-mem-warden.
 
+## Credits
+
+This gem is heavily based on the archived MIT-licensed `sidekiq-worker-killer` project.
+
 ## License
 
 The gem is available as open source under the terms of the GNU General Public License v2.0.

data/lib/sidekiq/mem/warden/version.rb

@@ -1,7 +1,7 @@
 module Sidekiq
   module Mem
-    module Warden
-      VERSION = "0.1.2"
+    class Warden
+      VERSION = "0.1.3"
     end
   end
 end

data/lib/sidekiq/mem/warden.rb

@@ -1,131 +1,119 @@
+require "get_process_mem"
 require "sidekiq"
+require "sidekiq/api"
+begin
+  require "sidekiq/middleware/modules"
+rescue LoadError
+end
 require "sidekiq/mem/warden/version"
 
 module Sidekiq
   module Mem
-    module Warden
-      class Error < StandardError; end
-
-      class Config
-        attr_accessor :memory_limit_mb, :check_interval, :quiet_timeout, :shutdown_timeout, :logger
-
-        def initialize
-          @memory_limit_mb = 1024
-          @check_interval = 15
-          @quiet_timeout = 300
-          @shutdown_timeout = 300
-          @logger = nil
+    class Warden
+      include Sidekiq::ServerMiddleware if defined?(Sidekiq::ServerMiddleware)
+
+      MUTEX = Mutex.new
+
+      def initialize(options = {})
+        @max_rss = options.fetch(:max_rss, 1024)
+        @grace_time = options.fetch(:grace_time, 300)
+        @shutdown_wait = options.fetch(:shutdown_wait, 30)
+        @kill_signal = options.fetch(:kill_signal, "SIGKILL")
+        @gc = options.fetch(:gc, true)
+        @skip_shutdown = options.fetch(:skip_shutdown_if, proc { false })
+        @on_shutdown = options.fetch(:on_shutdown, nil)
+      end
+
+      def call(worker, job, queue)
+        yield
+
+        return unless @max_rss.to_i > 0
+        return unless current_rss > @max_rss
+
+        GC.start(full_mark: true, immediate_sweep: true) if @gc
+        return unless current_rss > @max_rss
+
+        if skip_shutdown?(worker, job, queue)
+          warn "current RSS #{current_rss} exceeds maximum RSS #{@max_rss}, " \
+               "however shutdown will be ignored"
+          return
         end
+
+        warn "current RSS #{current_rss} of #{identity} exceeds maximum RSS #{@max_rss}"
+        run_shutdown_hook(worker, job, queue)
+        request_shutdown
       end
 
-      def self.configure
-        yield(config)
+      private
+
+      def run_shutdown_hook(worker, job, queue)
+        @on_shutdown.respond_to?(:call) && @on_shutdown.call(worker, job, queue)
+      end
+
+      def skip_shutdown?(worker, job, queue)
+        @skip_shutdown.respond_to?(:call) && @skip_shutdown.call(worker, job, queue)
       end
 
-      def self.install!(sidekiq_config = nil)
-        sidekiq_config ||= Sidekiq
-        sidekiq_config.on(:startup) do
-          start_monitor(Warden::Monitor.new(config))
+      def request_shutdown
+        Thread.new do
+          shutdown if MUTEX.try_lock
         end
       end
 
-      def self.config
-        @config ||= Config.new
+      def shutdown
+        warn "sending quiet to #{identity}"
+        sidekiq_process.quiet!
+
+        sleep(5)
+
+        warn "shutting down #{identity} in #{@grace_time} seconds"
+        wait_job_finish_in_grace_time
+
+        warn "stopping #{identity}"
+        sidekiq_process.stop!
+
+        warn "waiting #{@shutdown_wait} seconds before sending #{@kill_signal} to #{identity}"
+        sleep(@shutdown_wait)
+
+        warn "sending #{@kill_signal} to #{identity}"
+        ::Process.kill(@kill_signal, ::Process.pid)
       end
 
-      def self.start_monitor(monitor)
-        @start_mutex ||= Mutex.new
-        @start_mutex.synchronize do
-          return if @started
-          @started = true
-        end
-        monitor.start
+      def wait_job_finish_in_grace_time
+        start = Time.now
+        sleep(1) until grace_time_exceeded?(start) || jobs_finished?
       end
 
-      class Monitor
-        def initialize(config)
-          @config = config
-          @logger = config.logger || Sidekiq.logger
-          @triggered = false
-          @lock = Mutex.new
-          @start_lock = Mutex.new
-          @started = false
-        end
+      def grace_time_exceeded?(start)
+        return false if @grace_time == Float::INFINITY
 
-        def start
-          @start_lock.synchronize do
-            return if @started
-            @started = true
-          end
-
-          @thread = Thread.new do
-            Thread.current.name = "sidekiq-mem-warden" if Thread.current.respond_to?(:name=)
-            loop do
-              sleep @config.check_interval
-              next unless over_limit?
-              trigger!
-              break
-            end
-          end
-        end
+        start + @grace_time < Time.now
+      end
 
-        private
-
-        def trigger!
-          @lock.synchronize do
-            return if @triggered
-            @triggered = true
-          end
-
-          @logger.warn("[sidekiq-mem-warden] RSS over limit (#{rss_mb}MB >= #{@config.memory_limit_mb}MB), quieting")
-          begin
-            Process.kill("TSTP", Process.pid)
-          rescue StandardError => e
-            @logger.warn("[sidekiq-mem-warden] failed to send TSTP: #{e.class}: #{e.message}")
-          end
-
-          sleep @config.quiet_timeout
-          wait_for_idle(@config.shutdown_timeout)
-
-          @logger.warn("[sidekiq-mem-warden] shutting down for restart")
-          begin
-            Process.kill("TERM", Process.pid)
-          rescue StandardError => e
-            @logger.warn("[sidekiq-mem-warden] failed to send TERM: #{e.class}: #{e.message}")
-          end
-        end
+      def jobs_finished?
+        sidekiq_process.stopping? && sidekiq_process["busy"] == 0
+      end
 
-        def wait_for_idle(timeout_seconds)
-          deadline = Time.now + timeout_seconds
-          while Time.now < deadline
-            busy = Sidekiq::Workers.new.size
-            return if busy == 0
-            sleep 1
-          end
-          @logger.warn("[sidekiq-mem-warden] timeout waiting for busy to drain; proceeding")
-        end
+      def current_rss
+        ::GetProcessMem.new.mb
+      end
 
-        def over_limit?
-          rss_mb >= @config.memory_limit_mb
-        end
+      def sidekiq_process
+        Sidekiq::ProcessSet.new.find do |process|
+          process["identity"] == identity
+        end || raise("No sidekiq worker with identity #{identity} found")
+      end
 
-        def rss_mb
-          (rss_kb.to_f / 1024).round(2)
+      def identity
+        if respond_to?(:config) && config
+          config[:identity] || config["identity"]
+        else
+          Sidekiq.default_configuration[:identity] || Sidekiq.default_configuration["identity"]
         end
+      end
 
-        def rss_kb
-          status_path = "/proc/#{Process.pid}/status"
-          if File.exist?(status_path)
-            status = File.read(status_path)
-            match = status.match(/^VmRSS:\s+(\d+)\s+kB$/)
-            return match[1].to_i if match
-          end
-
-          output = `ps -o rss= -p #{Process.pid}`.to_s.strip
-          output.to_i
-        rescue StandardError
-          0
-        end
+      def warn(msg)
+        Sidekiq.logger.warn(msg)
       end
     end
   end

data/sidekiq-mem-warden.gemspec

@@ -36,7 +36,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "sidekiq", ">= 6.0"
+  spec.add_dependency "get_process_mem", ">= 0.2.0"
+  spec.add_dependency "sidekiq", ">= 7.0"
   spec.add_development_dependency "bundler", ">= 1.17"
   spec.add_development_dependency "rake", ">= 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-mem-warden
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - tedaford
@@ -9,20 +9,34 @@ bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: get_process_mem
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: sidekiq
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '6.0'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement