sidekiq-worker-killer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8a2cde25a88bee5182a6b202430223277271a183
+  data.tar.gz: 2c8b3c8908280b14e71417cbde9daef57b33ff71
+SHA512:
+  metadata.gz: f72ef844066d409eb417d2b0aa89b629994c621d74a1197076562012a420cad9eab04914492d1e2db9f5d06d684732f62f24a52f3f0608e627e74f25ace39e27
+  data.tar.gz: bac133d8b5a0ba793ab2bf8fa5a2378a2d20b74a06cfe4fe6c522ace3fa9f382701246485c3a87103cff84894f045d4cc8cf069ccc26d4c3c013d203fd5b36f0

data/README.md

@@ -0,0 +1,44 @@
+
+# sidekiq-worker-killer
+[![CircleCI](https://circleci.com/gh/klaxit/sidekiq-worker-killer.svg?style=shield&circle-token=:circle-token)](https://circleci.com/gh/klaxit/sidekiq-worker-killer)
+
+[Sidekiq](https://github.com/mperham/sidekiq) is probably the best background processing framework today. At the same time, memory leaks are very hard to tackle in Ruby and we often find ourselves with growing memory consumption.
+
+Highly inspired by [Gitlab Sidekiq MemoryKiller](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/sidekiq_middleware/shutdown.rb) and [Noxa Sidekiq killer](https://github.com/Noxa/sidekiq-killer).
+
+## Install
+Use [Bundler](http://bundler.io/)
+```
+gem "sidekiq-worker-killer", git: "https://github.com/klaxit/sidekiq-worker-killer"
+```
+
+## Usage
+
+Add this to your Sidekiq configuration.
+
+```
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add Sidekiq::WorkerKiller, max_rss: 250
+  end
+end
+```
+
+# Available options
+
+The following options can be overrided.
+
+| Option | Defaults | Description |
+| ------- | ------- | ----------- |
+| max_rss | 0 MB (disabled) | Max RSS in megabytes.                                              |
+| grace_time | 900 seconds | When a shutdown is triggered, the Sidekiq process will keep working normally for another 15 minutes. |
+| shutdown_wait | 30 seconds | When the grace time expires, existing jobs get 30 seconds to finish. After that, shutdown signal is triggered.  |
+| shutdown_signal | SIGKILL | Signal to use to shutdown sidekiq |
+
+## Authors
+
+See the list of [contributors](https://github.com/klaxit/sidekiq-worker-killer/contributors) who participated in this project.
+
+## License
+
+Please see LICENSE

data/lib/sidekiq/worker_killer.rb

@@ -0,0 +1,70 @@
+require "get_process_mem"
+require "sidekiq"
+
+module Sidekiq
+  # Sidekiq server middleware. Kill worker when the RSS memory exceeds limit
+  # after a given grace time.
+  class WorkerKiller
+    MUTEX = Mutex.new
+
+    def initialize(options = {})
+      @max_rss         = (options[:max_rss]         || 0)
+      @grace_time      = (options[:grace_time]      || 15 * 60)
+      @shutdown_wait   = (options[:shutdown_wait]   || 30)
+      @shutdown_signal = (options[:shutdown_signal] || "SIGKILL")
+    end
+
+    def call(worker, _job, _queue)
+      yield
+      # Skip if the max RSS is not exceeded
+      Sidekiq.logger.debug("current RSS is #{current_rss} MB")
+      return unless @max_rss > 0 && current_rss > @max_rss
+      # Perform kill
+      perform_kill(worker)
+    end
+
+    private
+
+    def perform_kill(worker)
+      # In another thread to allow undelying job to finish
+      Thread.new do
+        # Return if another thread is already waiting to shut Sidekiq down
+        return unless MUTEX.try_lock
+
+        # Perform the killing process
+        worker_ref = "[PID #{pid} - Worker #{worker.class}]"
+
+        warn "current RSS #{current_rss} exceeds maximum RSS #{@max_rss}"
+        warn "this thread will shut down #{worker_ref} in " \
+             "#{@grace_time} seconds"
+        sleep(@grace_time)
+
+        warn "sending SIGTERM to #{worker_ref}"
+        kill("SIGTERM", pid)
+
+        warn "waiting #{@shutdown_wait} seconds before sending " \
+             "#{@shutdown_signal} to #{worker_ref}"
+        sleep(@shutdown_wait)
+
+        warn "sending #{@shutdown_signal} to #{worker_ref}"
+        kill(@shutdown_signal, pid)
+      end
+    end
+
+    def current_rss
+      @current_rss ||= ::GetProcessMem.new.mb
+    end
+
+    def kill(signal, pid)
+      ::Process.kill(signal, pid)
+    end
+
+    def pid
+      @pid ||= ::Process.pid
+    end
+
+    def warn(msg)
+      Sidekiq.logger.warn(msg)
+    end
+  end
+end

data/lib/sidekiq/worker_killer/version.rb

@@ -0,0 +1,5 @@
+module Sidekiq
+  class WorkerKiller
+    VERSION = "0.1.0".freeze
+  end
+end

data/spec/sidekiq/worker_killer_spec.rb

@@ -0,0 +1,32 @@
+require "spec_helper"
+
+describe Sidekiq::WorkerKiller do
+  describe "#call" do
+    let(:worker){ double("worker") }
+    let(:job){ double("job") }
+    let(:queue){ double("queue") }
+    it "should yield" do
+      expect { |b|
+        subject.call(worker, job, queue, &b)
+      }.to yield_with_no_args
+    end
+    context "when current rss is over max rss" do
+      subject{ described_class.new(max_rss: 2) }
+      before do
+        allow(subject).to receive(:current_rss).and_return(3)
+      end
+      it "should perform kill" do
+        expect(subject).to receive(:perform_kill).with(worker)
+        subject.call(worker, job, queue){}
+      end
+      context "but max rss is 0" do
+        subject{ described_class.new(max_rss: 0) }
+        it "should not perform kill" do
+          expect(subject).to_not receive(:perform_kill).with(worker)
+          subject.call(worker, job, queue){}
+        end
+      end
+    end
+
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+
+require "rspec"
+require "sidekiq/worker_killer"
+
+RSpec.configure do |config|
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = "random"
+end

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: sidekiq-worker-killer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Cyrille Courtiere
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-04-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: get_process_mem
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.49.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.49.1
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+description: 
+email:
+- dev@klaxit.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/sidekiq/worker_killer.rb
+- lib/sidekiq/worker_killer/version.rb
+- spec/sidekiq/worker_killer_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/klaxit/sidekiq-worker-killer
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14
+signing_key: 
+specification_version: 4
+summary: Sidekiq worker killer
+test_files:
+- spec/sidekiq/worker_killer_spec.rb
+- spec/spec_helper.rb