sidekiq-worker-killer 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a2cde25a88bee5182a6b202430223277271a183
-  data.tar.gz: 2c8b3c8908280b14e71417cbde9daef57b33ff71
+  metadata.gz: 643766850e2b525cc9458efaeb259b2525654235
+  data.tar.gz: 25cc9a461b70258f9ce09ab71fa4d59f48150744
 SHA512:
-  metadata.gz: f72ef844066d409eb417d2b0aa89b629994c621d74a1197076562012a420cad9eab04914492d1e2db9f5d06d684732f62f24a52f3f0608e627e74f25ace39e27
-  data.tar.gz: bac133d8b5a0ba793ab2bf8fa5a2378a2d20b74a06cfe4fe6c522ace3fa9f382701246485c3a87103cff84894f045d4cc8cf069ccc26d4c3c013d203fd5b36f0
+  metadata.gz: 13e54d178b6eaf6de4c18bff3cb7bf5a0f583a2a511880514cb0b50c4158a8acf93dad622f3d906304e6b4a581c1d4a9b34a9e100354e0d3984891fb09ce34ec
+  data.tar.gz: e538edc1f1d17a2ffcba3121551bde5234375eb1648e98c1790efd973e1f8934131ad3ec9ecc8ace4101e2ec0ca97c62908e04b62b7b3243c6676a3511458d70

data/README.md

@@ -1,5 +1,6 @@
 
 # sidekiq-worker-killer
+[![Gem Version](https://badge.fury.io/rb/sidekiq-worker-killer.svg)](https://badge.fury.io/rb/sidekiq-worker-killer)
 [![CircleCI](https://circleci.com/gh/klaxit/sidekiq-worker-killer.svg?style=shield&circle-token=:circle-token)](https://circleci.com/gh/klaxit/sidekiq-worker-killer)
 
 [Sidekiq](https://github.com/mperham/sidekiq) is probably the best background processing framework today. At the same time, memory leaks are very hard to tackle in Ruby and we often find ourselves with growing memory consumption.
@@ -9,7 +10,7 @@ Highly inspired by [Gitlab Sidekiq MemoryKiller](https://gitlab.com/gitlab-org/g
 ## Install
 Use [Bundler](http://bundler.io/)
 ```
-gem "sidekiq-worker-killer", git: "https://github.com/klaxit/sidekiq-worker-killer"
+gem "sidekiq-worker-killer"
 ```
 
 ## Usage
@@ -19,7 +20,7 @@ Add this to your Sidekiq configuration.
 ```
 Sidekiq.configure_server do |config|
   config.server_middleware do |chain|
-    chain.add Sidekiq::WorkerKiller, max_rss: 250
+    chain.add Sidekiq::WorkerKiller, max_rss: 480
   end
 end
 ```
@@ -30,10 +31,10 @@ The following options can be overrided.
 
 | Option | Defaults | Description |
 | ------- | ------- | ----------- |
-| max_rss | 0 MB (disabled) | Max RSS in megabytes.                                              |
-| grace_time | 900 seconds | When a shutdown is triggered, the Sidekiq process will keep working normally for another 15 minutes. |
-| shutdown_wait | 30 seconds | When the grace time expires, existing jobs get 30 seconds to finish. After that, shutdown signal is triggered.  |
-| shutdown_signal | SIGKILL | Signal to use to shutdown sidekiq |
+| max_rss | 0 MB (disabled) | max RSS in megabytes. Above this, shutdown will be triggered. |
+| grace_time | 900 seconds | when shutdown is triggered, the Sidekiq process will not accept new job but wait 15 minutes for running jobs to finish.  |
+| shutdown_wait | 30 seconds | when the grace time expires, still running jobs get 30 seconds to terminate. After that, kill signal is triggered.  |
+| kill_signal | SIGKILL | Signal to use kill Sidekiq process if it doesn't terminate.  |
 
 ## Authors
 

data/lib/sidekiq/worker_killer.rb

@@ -1,66 +1,83 @@
 require "get_process_mem"
 require "sidekiq"
+require "sidekiq/util"
 
 module Sidekiq
   # Sidekiq server middleware. Kill worker when the RSS memory exceeds limit
   # after a given grace time.
   class WorkerKiller
+    include Sidekiq::Util
+
     MUTEX = Mutex.new
 
     def initialize(options = {})
       @max_rss         = (options[:max_rss]         || 0)
       @grace_time      = (options[:grace_time]      || 15 * 60)
       @shutdown_wait   = (options[:shutdown_wait]   || 30)
-      @shutdown_signal = (options[:shutdown_signal] || "SIGKILL")
+      @kill_signal     = (options[:kill_signal]     || "SIGKILL")
     end
 
-    def call(worker, _job, _queue)
+    def call(_worker, _job, _queue)
       yield
       # Skip if the max RSS is not exceeded
-      Sidekiq.logger.debug("current RSS is #{current_rss} MB")
       return unless @max_rss > 0 && current_rss > @max_rss
-      # Perform kill
-      perform_kill(worker)
+      # Launch the shutdown process
+      warn "current RSS #{current_rss} of #{identity} exceeds " \
+           "maximum RSS #{@max_rss}"
+      request_shutdown
     end
 
     private
 
-    def perform_kill(worker)
+    def request_shutdown
       # In another thread to allow undelying job to finish
       Thread.new do
-        # Return if another thread is already waiting to shut Sidekiq down
-        return unless MUTEX.try_lock
+        # Only if another thread is not already
+        # shutting down the Sidekiq process
+        shutdown if MUTEX.try_lock
+      end
+    end
 
-        # Perform the killing process
-        worker_ref = "[PID #{pid} - Worker #{worker.class}]"
+    def shutdown
+      warn "sending #{quiet_signal} to #{identity}"
+      signal(quiet_signal, pid)
 
-        warn "current RSS #{current_rss} exceeds maximum RSS #{@max_rss}"
-        warn "this thread will shut down #{worker_ref} in " \
-             "#{@grace_time} seconds"
-        sleep(@grace_time)
+      warn "shutting down #{identity} in #{@grace_time} seconds"
+      sleep(@grace_time)
 
-        warn "sending SIGTERM to #{worker_ref}"
-        kill("SIGTERM", pid)
+      warn "sending SIGTERM to #{identity}"
+      signal("SIGTERM", pid)
 
-        warn "waiting #{@shutdown_wait} seconds before sending " \
-             "#{@shutdown_signal} to #{worker_ref}"
-        sleep(@shutdown_wait)
+      warn "waiting #{@shutdown_wait} seconds before sending " \
+            "#{@kill_signal} to #{identity}"
+      sleep(@kill_signal)
 
-        warn "sending #{@shutdown_signal} to #{worker_ref}"
-        kill(@shutdown_signal, pid)
-      end
+      warn "sending #{@kill_signal} to #{identity}"
+      signal(@kill_signal, pid)
     end
 
     def current_rss
-      @current_rss ||= ::GetProcessMem.new.mb
+      ::GetProcessMem.new.mb
     end
 
-    def kill(signal, pid)
+    def signal(signal, pid)
       ::Process.kill(signal, pid)
     end
 
     def pid
-      @pid ||= ::Process.pid
+      ::Process.pid
+    end
+
+    def identity
+      "#{hostname}:#{pid}"
+    end
+
+    def quiet_signal
+      if Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new("5.0")
+        "TSTP"
+      else
+        "USR1"
+      end
     end
 
     def warn(msg)

data/lib/sidekiq/worker_killer/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   class WorkerKiller
-    VERSION = "0.1.0".freeze
+    VERSION = "0.2.0".freeze
   end
 end

data/spec/sidekiq/worker_killer_spec.rb

@@ -1,6 +1,11 @@
 require "spec_helper"
 
 describe Sidekiq::WorkerKiller do
+
+  before do
+    allow(subject).to receive(:warn) # silence "warn" logs
+  end
+
   describe "#call" do
     let(:worker){ double("worker") }
     let(:job){ double("job") }
@@ -15,18 +20,44 @@ describe Sidekiq::WorkerKiller do
       before do
         allow(subject).to receive(:current_rss).and_return(3)
       end
-      it "should perform kill" do
-        expect(subject).to receive(:perform_kill).with(worker)
+      it "should request shutdown" do
+        expect(subject).to receive(:request_shutdown)
         subject.call(worker, job, queue){}
       end
       context "but max rss is 0" do
         subject{ described_class.new(max_rss: 0) }
-        it "should not perform kill" do
-          expect(subject).to_not receive(:perform_kill).with(worker)
+        it "should not request shutdown" do
+          expect(subject).to_not receive(:request_shutdown)
           subject.call(worker, job, queue){}
         end
       end
     end
+  end
+
+  describe "#request_shutdown" do
+    before { allow(subject).to receive(:shutdown){ sleep 0.01 } }
+    it "should call shutdown" do
+      expect(subject).to receive(:shutdown)
+      subject.send(:request_shutdown).join
+    end
+    it "should not call shutdown twice when called concurrently" do
+      expect(subject).to receive(:shutdown).once
+      2.times.map{ subject.send(:request_shutdown) }.each(&:join)
+    end
+  end
 
+  describe "#quiet_signal" do
+    it "should give TSTP if Sidekiq version is > 5.0" do
+      stub_const("Sidekiq::VERSION", "5.0")
+      expect(subject.send :quiet_signal).to eq "TSTP"
+      stub_const("Sidekiq::VERSION", "5.2.1")
+      expect(subject.send :quiet_signal).to eq "TSTP"
+    end
+    it "should give USR1 if Sidekiq version is < 5.0" do
+      stub_const("Sidekiq::VERSION", "3.0")
+      expect(subject.send :quiet_signal).to eq "USR1"
+      stub_const("Sidekiq::VERSION", "4.6.7")
+      expect(subject.send :quiet_signal).to eq "USR1"
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-worker-killer
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Cyrille Courtiere
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-17 00:00:00.000000000 Z
+date: 2018-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: get_process_mem