sidekiq-worker-killer 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +44 -0
- data/lib/sidekiq/worker_killer.rb +70 -0
- data/lib/sidekiq/worker_killer/version.rb +5 -0
- data/spec/sidekiq/worker_killer_spec.rb +32 -0
- data/spec/spec_helper.rb +13 -0
- metadata +106 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 8a2cde25a88bee5182a6b202430223277271a183
|
4
|
+
data.tar.gz: 2c8b3c8908280b14e71417cbde9daef57b33ff71
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: f72ef844066d409eb417d2b0aa89b629994c621d74a1197076562012a420cad9eab04914492d1e2db9f5d06d684732f62f24a52f3f0608e627e74f25ace39e27
|
7
|
+
data.tar.gz: bac133d8b5a0ba793ab2bf8fa5a2378a2d20b74a06cfe4fe6c522ace3fa9f382701246485c3a87103cff84894f045d4cc8cf069ccc26d4c3c013d203fd5b36f0
|
data/README.md
ADDED
@@ -0,0 +1,44 @@
|
|
1
|
+
|
2
|
+
# sidekiq-worker-killer
|
3
|
+
[![CircleCI](https://circleci.com/gh/klaxit/sidekiq-worker-killer.svg?style=shield&circle-token=:circle-token)](https://circleci.com/gh/klaxit/sidekiq-worker-killer)
|
4
|
+
|
5
|
+
[Sidekiq](https://github.com/mperham/sidekiq) is probably the best background processing framework today. At the same time, memory leaks are very hard to tackle in Ruby and we often find ourselves with growing memory consumption.
|
6
|
+
|
7
|
+
Highly inspired by [Gitlab Sidekiq MemoryKiller](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/sidekiq_middleware/shutdown.rb) and [Noxa Sidekiq killer](https://github.com/Noxa/sidekiq-killer).
|
8
|
+
|
9
|
+
## Install
|
10
|
+
Use [Bundler](http://bundler.io/)
|
11
|
+
```
|
12
|
+
gem "sidekiq-worker-killer", git: "https://github.com/klaxit/sidekiq-worker-killer"
|
13
|
+
```
|
14
|
+
|
15
|
+
## Usage
|
16
|
+
|
17
|
+
Add this to your Sidekiq configuration.
|
18
|
+
|
19
|
+
```
|
20
|
+
Sidekiq.configure_server do |config|
|
21
|
+
config.server_middleware do |chain|
|
22
|
+
chain.add Sidekiq::WorkerKiller, max_rss: 250
|
23
|
+
end
|
24
|
+
end
|
25
|
+
```
|
26
|
+
|
27
|
+
# Available options
|
28
|
+
|
29
|
+
The following options can be overrided.
|
30
|
+
|
31
|
+
| Option | Defaults | Description |
|
32
|
+
| ------- | ------- | ----------- |
|
33
|
+
| max_rss | 0 MB (disabled) | Max RSS in megabytes. |
|
34
|
+
| grace_time | 900 seconds | When a shutdown is triggered, the Sidekiq process will keep working normally for another 15 minutes. |
|
35
|
+
| shutdown_wait | 30 seconds | When the grace time expires, existing jobs get 30 seconds to finish. After that, shutdown signal is triggered. |
|
36
|
+
| shutdown_signal | SIGKILL | Signal to use to shutdown sidekiq |
|
37
|
+
|
38
|
+
## Authors
|
39
|
+
|
40
|
+
See the list of [contributors](https://github.com/klaxit/sidekiq-worker-killer/contributors) who participated in this project.
|
41
|
+
|
42
|
+
## License
|
43
|
+
|
44
|
+
Please see LICENSE
|
@@ -0,0 +1,70 @@
|
|
1
|
+
require "get_process_mem"
|
2
|
+
require "sidekiq"
|
3
|
+
|
4
|
+
module Sidekiq
|
5
|
+
# Sidekiq server middleware. Kill worker when the RSS memory exceeds limit
|
6
|
+
# after a given grace time.
|
7
|
+
class WorkerKiller
|
8
|
+
MUTEX = Mutex.new
|
9
|
+
|
10
|
+
def initialize(options = {})
|
11
|
+
@max_rss = (options[:max_rss] || 0)
|
12
|
+
@grace_time = (options[:grace_time] || 15 * 60)
|
13
|
+
@shutdown_wait = (options[:shutdown_wait] || 30)
|
14
|
+
@shutdown_signal = (options[:shutdown_signal] || "SIGKILL")
|
15
|
+
end
|
16
|
+
|
17
|
+
def call(worker, _job, _queue)
|
18
|
+
yield
|
19
|
+
# Skip if the max RSS is not exceeded
|
20
|
+
Sidekiq.logger.debug("current RSS is #{current_rss} MB")
|
21
|
+
return unless @max_rss > 0 && current_rss > @max_rss
|
22
|
+
# Perform kill
|
23
|
+
perform_kill(worker)
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def perform_kill(worker)
|
29
|
+
# In another thread to allow undelying job to finish
|
30
|
+
Thread.new do
|
31
|
+
# Return if another thread is already waiting to shut Sidekiq down
|
32
|
+
return unless MUTEX.try_lock
|
33
|
+
|
34
|
+
# Perform the killing process
|
35
|
+
worker_ref = "[PID #{pid} - Worker #{worker.class}]"
|
36
|
+
|
37
|
+
warn "current RSS #{current_rss} exceeds maximum RSS #{@max_rss}"
|
38
|
+
warn "this thread will shut down #{worker_ref} in " \
|
39
|
+
"#{@grace_time} seconds"
|
40
|
+
sleep(@grace_time)
|
41
|
+
|
42
|
+
warn "sending SIGTERM to #{worker_ref}"
|
43
|
+
kill("SIGTERM", pid)
|
44
|
+
|
45
|
+
warn "waiting #{@shutdown_wait} seconds before sending " \
|
46
|
+
"#{@shutdown_signal} to #{worker_ref}"
|
47
|
+
sleep(@shutdown_wait)
|
48
|
+
|
49
|
+
warn "sending #{@shutdown_signal} to #{worker_ref}"
|
50
|
+
kill(@shutdown_signal, pid)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def current_rss
|
55
|
+
@current_rss ||= ::GetProcessMem.new.mb
|
56
|
+
end
|
57
|
+
|
58
|
+
def kill(signal, pid)
|
59
|
+
::Process.kill(signal, pid)
|
60
|
+
end
|
61
|
+
|
62
|
+
def pid
|
63
|
+
@pid ||= ::Process.pid
|
64
|
+
end
|
65
|
+
|
66
|
+
def warn(msg)
|
67
|
+
Sidekiq.logger.warn(msg)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe Sidekiq::WorkerKiller do
|
4
|
+
describe "#call" do
|
5
|
+
let(:worker){ double("worker") }
|
6
|
+
let(:job){ double("job") }
|
7
|
+
let(:queue){ double("queue") }
|
8
|
+
it "should yield" do
|
9
|
+
expect { |b|
|
10
|
+
subject.call(worker, job, queue, &b)
|
11
|
+
}.to yield_with_no_args
|
12
|
+
end
|
13
|
+
context "when current rss is over max rss" do
|
14
|
+
subject{ described_class.new(max_rss: 2) }
|
15
|
+
before do
|
16
|
+
allow(subject).to receive(:current_rss).and_return(3)
|
17
|
+
end
|
18
|
+
it "should perform kill" do
|
19
|
+
expect(subject).to receive(:perform_kill).with(worker)
|
20
|
+
subject.call(worker, job, queue){}
|
21
|
+
end
|
22
|
+
context "but max rss is 0" do
|
23
|
+
subject{ described_class.new(max_rss: 0) }
|
24
|
+
it "should not perform kill" do
|
25
|
+
expect(subject).to_not receive(:perform_kill).with(worker)
|
26
|
+
subject.call(worker, job, queue){}
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
2
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
|
3
|
+
|
4
|
+
require "rspec"
|
5
|
+
require "sidekiq/worker_killer"
|
6
|
+
|
7
|
+
RSpec.configure do |config|
|
8
|
+
# Run specs in random order to surface order dependencies. If you find an
|
9
|
+
# order dependency and want to debug it, you can fix the order by providing
|
10
|
+
# the seed, which is printed after each run.
|
11
|
+
# --seed 1234
|
12
|
+
config.order = "random"
|
13
|
+
end
|
metadata
ADDED
@@ -0,0 +1,106 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: sidekiq-worker-killer
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Cyrille Courtiere
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2018-04-17 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: get_process_mem
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.2.1
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.2.1
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: sidekiq
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '3'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '3'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rubocop
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.49.1
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.49.1
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rspec
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '3.5'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '3.5'
|
69
|
+
description:
|
70
|
+
email:
|
71
|
+
- dev@klaxit.com
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- README.md
|
77
|
+
- lib/sidekiq/worker_killer.rb
|
78
|
+
- lib/sidekiq/worker_killer/version.rb
|
79
|
+
- spec/sidekiq/worker_killer_spec.rb
|
80
|
+
- spec/spec_helper.rb
|
81
|
+
homepage: http://github.com/klaxit/sidekiq-worker-killer
|
82
|
+
licenses: []
|
83
|
+
metadata: {}
|
84
|
+
post_install_message:
|
85
|
+
rdoc_options: []
|
86
|
+
require_paths:
|
87
|
+
- lib
|
88
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
89
|
+
requirements:
|
90
|
+
- - ">="
|
91
|
+
- !ruby/object:Gem::Version
|
92
|
+
version: '0'
|
93
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - ">="
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: '0'
|
98
|
+
requirements: []
|
99
|
+
rubyforge_project:
|
100
|
+
rubygems_version: 2.6.14
|
101
|
+
signing_key:
|
102
|
+
specification_version: 4
|
103
|
+
summary: Sidekiq worker killer
|
104
|
+
test_files:
|
105
|
+
- spec/sidekiq/worker_killer_spec.rb
|
106
|
+
- spec/spec_helper.rb
|