sidekiq-worker-killer 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4269c4f6da1a1eff0beed38c5ced2f6fca035b83
-  data.tar.gz: 9d9fb7cb1bf97982dfb489ab5a397b06930c0620
+  metadata.gz: c2b88a4d00042d82a27ebd81fa27832f01873618
+  data.tar.gz: 010d2676048b457204d403509774c4e013cd39d4
 SHA512:
-  metadata.gz: 716ea8ce12527e8653f0bc08c9cc16acc428917340907a9701e2e45d8f3996a157dacc634a96d0db74db3dfa949ba8ffb0377391837942df5d343d5b1870463f
-  data.tar.gz: 0f04a39f3ac416f231e7b075a34ce827f4c435183e1b2277d50ce1fb4392b66a7ddcef59c60d4c0e1392ac34c572849501624d8be642b3cb76495562ebd85cd5
+  metadata.gz: 9aef7ae408f33ed04a2cbe81ba673887d877762c803482da841a25c0878b2c91e96e6eb8b4eabfddcbcfb42b8bf7ce56e218463df867af44fadad7afb4a6a0c9
+  data.tar.gz: 4c9cfa47bdb45445f6647ca0297e3118f4f37cc675759a3e292a96e701577c385c6f7bcce832cd8c5082f59acce4449aefad7e92160e59f8049eefa5499d95c7

data/README.md

@@ -3,13 +3,15 @@
 [![Gem Version](https://badge.fury.io/rb/sidekiq-worker-killer.svg)](https://badge.fury.io/rb/sidekiq-worker-killer)
 [![CircleCI](https://circleci.com/gh/klaxit/sidekiq-worker-killer.svg?style=shield&circle-token=:circle-token)](https://circleci.com/gh/klaxit/sidekiq-worker-killer)
 
-[Sidekiq](https://github.com/mperham/sidekiq) is probably the best background processing framework today. At the same time, memory leaks are very hard to tackle in Ruby and we often find ourselves with growing memory consumption.
+[Sidekiq](https://github.com/mperham/sidekiq) is probably the best background processing framework today. At the same time, memory leaks are very hard to tackle in Ruby and we often find ourselves with growing memory consumption. Instead of spending herculean effort fixing leaks, why not kill your processes when they got to be too large?
 
 Highly inspired by [Gitlab Sidekiq MemoryKiller](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/sidekiq_middleware/shutdown.rb) and [Noxa Sidekiq killer](https://github.com/Noxa/sidekiq-killer).
 
+quick-refs: [install](#install) | [usage](#usage) | [available options](#available-options) | [development](#development)
+
 ## Install
 Use [Bundler](http://bundler.io/)
-```
+```ruby
 gem "sidekiq-worker-killer"
 ```
 
@@ -17,7 +19,9 @@ gem "sidekiq-worker-killer"
 
 Add this to your Sidekiq configuration.
 
-```
+```ruby
+require 'sidekiq/worker_killer'
+
 Sidekiq.configure_server do |config|
   config.server_middleware do |chain|
     chain.add Sidekiq::WorkerKiller, max_rss: 480
@@ -25,16 +29,30 @@ Sidekiq.configure_server do |config|
 end
 ```
 
-# Available options
+## Available options
 
 The following options can be overrided.
 
 | Option | Defaults | Description |
 | ------- | ------- | ----------- |
 | max_rss | 0 MB (disabled) | max RSS in megabytes. Above this, shutdown will be triggered. |
-| grace_time | 900 seconds | when shutdown is triggered, the Sidekiq process will not accept new job and wait at most 15 minutes for running jobs to finish.  |
+| grace_time | 900 seconds | when shutdown is triggered, the Sidekiq process will not accept new job and wait at most 15 minutes for running jobs to finish. If Float::INFINITY specified, will wait forever  |
 | shutdown_wait | 30 seconds | when the grace time expires, still running jobs get 30 seconds to terminate. After that, kill signal is triggered.  |
 | kill_signal | SIGKILL | Signal to use kill Sidekiq process if it doesn't terminate.  |
+| gc | true | Try to run garbage collection before Sidekiq process terminate in case of max_rss exceeded  |
+
+## Development
+
+Pull Requests are very welcome!
+
+There are tasks that may help you along the way in a makefile:
+
+```bash
+make console # Loads the whole stack in an IRB session.
+make test # Run tests.
+make lint # Run rubocop linter.
+```
+Please make sure that you have tested your code carefully before opening a PR, and make sure as well that you have no style issues.
 
 ## Authors
 

data/lib/sidekiq/worker_killer.rb

@@ -2,103 +2,110 @@ require "get_process_mem"
 require "sidekiq"
 require "sidekiq/util"
 
-module Sidekiq
-  # Sidekiq server middleware. Kill worker when the RSS memory exceeds limit
-  # after a given grace time.
-  class WorkerKiller
-    include Sidekiq::Util
-
-    MUTEX = Mutex.new
-
-    def initialize(options = {})
-      @max_rss         = (options[:max_rss]         || 0)
-      @grace_time      = (options[:grace_time]      || 15 * 60)
-      @shutdown_wait   = (options[:shutdown_wait]   || 30)
-      @kill_signal     = (options[:kill_signal]     || "SIGKILL")
-    end
+# Sidekiq server middleware. Kill worker when the RSS memory exceeds limit
+# after a given grace time.
+class Sidekiq::WorkerKiller
+  include Sidekiq::Util
+
+  MUTEX = Mutex.new
+
+  def initialize(options = {})
+    @max_rss         = options.fetch(:max_rss, 0)
+    @grace_time      = options.fetch(:grace_time, 15 * 60)
+    @shutdown_wait   = options.fetch(:shutdown_wait, 30)
+    @kill_signal     = options.fetch(:kill_signal, "SIGKILL")
+    @gc              = options.fetch(:gc, true)
+  end
 
-    def call(_worker, _job, _queue)
-      yield
-      # Skip if the max RSS is not exceeded
-      return unless @max_rss > 0 && current_rss > @max_rss
-      GC.start(full_mark: true, immediate_sweep: true)
-      return unless @max_rss > 0 && current_rss > @max_rss
-      # Launch the shutdown process
-      warn "current RSS #{current_rss} of #{identity} exceeds " \
-           "maximum RSS #{@max_rss}"
-      request_shutdown
-    end
+  def call(_worker, _job, _queue)
+    yield
+    # Skip if the max RSS is not exceeded
+    return unless @max_rss > 0
+    return unless current_rss > @max_rss
+    GC.start(full_mark: true, immediate_sweep: true) if @gc
+    return unless current_rss > @max_rss
+    # Launch the shutdown process
+    warn "current RSS #{current_rss} of #{identity} exceeds " \
+         "maximum RSS #{@max_rss}"
+    request_shutdown
+  end
 
-    private
+  private
 
-    def request_shutdown
-      # In another thread to allow undelying job to finish
-      Thread.new do
-        # Only if another thread is not already
-        # shutting down the Sidekiq process
-        shutdown if MUTEX.try_lock
-      end
+  def request_shutdown
+    # In another thread to allow undelying job to finish
+    Thread.new do
+      # Only if another thread is not already
+      # shutting down the Sidekiq process
+      shutdown if MUTEX.try_lock
     end
+  end
 
-    def shutdown
-      warn "sending #{quiet_signal} to #{identity}"
-      signal(quiet_signal, pid)
+  def shutdown
+    warn "sending #{quiet_signal} to #{identity}"
+    signal(quiet_signal, pid)
 
-      warn "shutting down #{identity} in #{@grace_time} seconds"
-      wait_job_finish_in_grace_time
+    warn "shutting down #{identity} in #{@grace_time} seconds"
+    wait_job_finish_in_grace_time
 
-      warn "sending SIGTERM to #{identity}"
-      signal("SIGTERM", pid)
+    warn "sending SIGTERM to #{identity}"
+    signal("SIGTERM", pid)
 
-      warn "waiting #{@shutdown_wait} seconds before sending " \
-            "#{@kill_signal} to #{identity}"
-      sleep(@shutdown_wait)
+    warn "waiting #{@shutdown_wait} seconds before sending " \
+          "#{@kill_signal} to #{identity}"
+    sleep(@shutdown_wait)
 
-      warn "sending #{@kill_signal} to #{identity}"
-      signal(@kill_signal, pid)
-    end
+    warn "sending #{@kill_signal} to #{identity}"
+    signal(@kill_signal, pid)
+  end
 
-    def wait_job_finish_in_grace_time
-      start = Time.now
-      loop do
-        break if start + @grace_time < Time.now || no_jobs_on_quiet_processes?
-        sleep(1)
-      end
+  def wait_job_finish_in_grace_time
+    start = Time.now
+    loop do
+      break if grace_time_exceeded?(start)
+      break if no_jobs_on_quiet_processes?
+      sleep(1)
     end
+  end
 
-    def no_jobs_on_quiet_processes?
-      Sidekiq::ProcessSet.new.each do |process|
-        return false if !process["busy"].zero? && process["quiet"]
-      end
-      true
-    end
+  def grace_time_exceeded?(start)
+    return false if @grace_time == Float::INFINITY
 
-    def current_rss
-      ::GetProcessMem.new.mb
-    end
+    start + @grace_time < Time.now
+  end
 
-    def signal(signal, pid)
-      ::Process.kill(signal, pid)
+  def no_jobs_on_quiet_processes?
+    Sidekiq::ProcessSet.new.each do |process|
+      return false if process["busy"] != 0 && process["quiet"] == "true"
     end
+    true
+  end
 
-    def pid
-      ::Process.pid
-    end
+  def current_rss
+    ::GetProcessMem.new.mb
+  end
 
-    def identity
-      "#{hostname}:#{pid}"
-    end
+  def signal(signal, pid)
+    ::Process.kill(signal, pid)
+  end
 
-    def quiet_signal
-      if Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new("5.0")
-        "TSTP"
-      else
-        "USR1"
-      end
-    end
+  def pid
+    ::Process.pid
+  end
+
+  def identity
+    "#{hostname}:#{pid}"
+  end
 
-    def warn(msg)
-      Sidekiq.logger.warn(msg)
+  def quiet_signal
+    if Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new("5.0")
+      "TSTP"
+    else
+      "USR1"
     end
   end
+
+  def warn(msg)
+    Sidekiq.logger.warn(msg)
+  end
 end

data/lib/sidekiq/worker_killer/version.rb

@@ -1,5 +1,7 @@
+# rubocop:disable Style/ClassAndModuleChildren
 module Sidekiq
   class WorkerKiller
-    VERSION = "0.3.0".freeze
+    VERSION = "0.4.0".freeze
   end
 end
+# rubocop:enable Style/ClassAndModuleChildren

data/spec/sidekiq/worker_killer_spec.rb

@@ -24,6 +24,19 @@ describe Sidekiq::WorkerKiller do
         expect(subject).to receive(:request_shutdown)
         subject.call(worker, job, queue){}
       end
+      it "should call garbage collect" do
+        allow(subject).to receive(:request_shutdown)
+        expect(GC).to receive(:start).with(full_mark: true, immediate_sweep: true)
+        subject.call(worker, job, queue){}
+      end
+      context "when gc is false" do
+        subject{ described_class.new(max_rss: 2, gc: false) }
+        it "should not call garbage collect" do
+          allow(subject).to receive(:request_shutdown)
+          expect(GC).not_to receive(:start)
+          subject.call(worker, job, queue){}
+        end
+      end
       context "but max rss is 0" do
         subject{ described_class.new(max_rss: 0) }
         it "should not request shutdown" do
@@ -35,14 +48,63 @@ describe Sidekiq::WorkerKiller do
   end
 
   describe "#request_shutdown" do
-    before { allow(subject).to receive(:shutdown){ sleep 0.01 } }
-    it "should call shutdown" do
-      expect(subject).to receive(:shutdown)
-      subject.send(:request_shutdown).join
+    context "grace time is default" do
+      before { allow(subject).to receive(:shutdown){ sleep 0.01 } }
+      it "should call shutdown" do
+        expect(subject).to receive(:shutdown)
+        subject.send(:request_shutdown).join
+      end
+      it "should not call shutdown twice when called concurrently" do
+        expect(subject).to receive(:shutdown).once
+        2.times.map{ subject.send(:request_shutdown) }.each(&:join)
+      end
+    end
+
+    context "grace time is 5 seconds" do
+      subject{ described_class.new(max_rss: 2, grace_time: 5.0, shutdown_wait: 0) }
+      it "should wait the specified grace time before calling shutdown" do
+        # there are busy jobs that will not terminate within the grace time
+        allow(subject).to receive(:no_jobs_on_quiet_processes?).and_return(false)
+
+        shutdown_request_time     = nil
+        shutdown_time             = nil
+
+        # replace the original #request_shutdown to track
+        # when the shutdown is requested
+        original_request_shutdown = subject.method(:request_shutdown)
+        allow(subject).to receive(:request_shutdown) do
+          shutdown_request_time= Time.now
+          original_request_shutdown.call
+        end
+
+        # track when the SIGTERM signal is sent
+        allow(Process).to receive(:kill) do |*args|
+          shutdown_time = Time.now if args[0] == 'SIGTERM'
+        end
+
+        allow(subject).to receive(:pid).and_return(99)
+
+        subject.send(:request_shutdown).join
+
+        elapsed_time = shutdown_time - shutdown_request_time
+
+        # the elapsed time beetween shutdown request and the actual
+        # shutdown signal should be greater than the specificed grace_time
+        expect(elapsed_time).to be >= 5.0
+      end
     end
-    it "should not call shutdown twice when called concurrently" do
-      expect(subject).to receive(:shutdown).once
-      2.times.map{ subject.send(:request_shutdown) }.each(&:join)
+
+    context "grace time is Float::INFINITY" do
+      subject{ described_class.new(max_rss: 2, grace_time: Float::INFINITY, shutdown_wait: 0) }
+      it "call signal only on jobs" do
+        allow(subject).to receive(:no_jobs_on_quiet_processes?).and_return(true)
+        allow(subject).to receive(:pid).and_return(99)
+        expect(Process).to receive(:kill).with('TSTP', 99)
+        expect(Process).to receive(:kill).with('SIGTERM', 99)
+        expect(Process).to receive(:kill).with('SIGKILL', 99)
+
+        subject.send(:request_shutdown).join
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-worker-killer
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Cyrille Courtiere
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-12 00:00:00.000000000 Z
+date: 2019-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: get_process_mem
@@ -39,33 +39,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3'
 - !ruby/object:Gem::Dependency
-  name: rubocop
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.49.1
+        version: '3.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.49.1
+        version: '3.5'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: 0.49.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: 0.49.1
 description: 
 email:
 - dev@klaxit.com