sidekiq-worker-killer 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +7 -6
- data/lib/sidekiq/worker_killer.rb +42 -25
- data/lib/sidekiq/worker_killer/version.rb +1 -1
- data/spec/sidekiq/worker_killer_spec.rb +35 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 643766850e2b525cc9458efaeb259b2525654235
|
4
|
+
data.tar.gz: 25cc9a461b70258f9ce09ab71fa4d59f48150744
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 13e54d178b6eaf6de4c18bff3cb7bf5a0f583a2a511880514cb0b50c4158a8acf93dad622f3d906304e6b4a581c1d4a9b34a9e100354e0d3984891fb09ce34ec
|
7
|
+
data.tar.gz: e538edc1f1d17a2ffcba3121551bde5234375eb1648e98c1790efd973e1f8934131ad3ec9ecc8ace4101e2ec0ca97c62908e04b62b7b3243c6676a3511458d70
|
data/README.md
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
|
2
2
|
# sidekiq-worker-killer
|
3
|
+
[![Gem Version](https://badge.fury.io/rb/sidekiq-worker-killer.svg)](https://badge.fury.io/rb/sidekiq-worker-killer)
|
3
4
|
[![CircleCI](https://circleci.com/gh/klaxit/sidekiq-worker-killer.svg?style=shield&circle-token=:circle-token)](https://circleci.com/gh/klaxit/sidekiq-worker-killer)
|
4
5
|
|
5
6
|
[Sidekiq](https://github.com/mperham/sidekiq) is probably the best background processing framework today. At the same time, memory leaks are very hard to tackle in Ruby and we often find ourselves with growing memory consumption.
|
@@ -9,7 +10,7 @@ Highly inspired by [Gitlab Sidekiq MemoryKiller](https://gitlab.com/gitlab-org/g
|
|
9
10
|
## Install
|
10
11
|
Use [Bundler](http://bundler.io/)
|
11
12
|
```
|
12
|
-
gem "sidekiq-worker-killer"
|
13
|
+
gem "sidekiq-worker-killer"
|
13
14
|
```
|
14
15
|
|
15
16
|
## Usage
|
@@ -19,7 +20,7 @@ Add this to your Sidekiq configuration.
|
|
19
20
|
```
|
20
21
|
Sidekiq.configure_server do |config|
|
21
22
|
config.server_middleware do |chain|
|
22
|
-
chain.add Sidekiq::WorkerKiller, max_rss:
|
23
|
+
chain.add Sidekiq::WorkerKiller, max_rss: 480
|
23
24
|
end
|
24
25
|
end
|
25
26
|
```
|
@@ -30,10 +31,10 @@ The following options can be overrided.
|
|
30
31
|
|
31
32
|
| Option | Defaults | Description |
|
32
33
|
| ------- | ------- | ----------- |
|
33
|
-
| max_rss | 0 MB (disabled) |
|
34
|
-
| grace_time | 900 seconds |
|
35
|
-
| shutdown_wait | 30 seconds |
|
36
|
-
|
|
34
|
+
| max_rss | 0 MB (disabled) | max RSS in megabytes. Above this, shutdown will be triggered. |
|
35
|
+
| grace_time | 900 seconds | when shutdown is triggered, the Sidekiq process will not accept new job but wait 15 minutes for running jobs to finish. |
|
36
|
+
| shutdown_wait | 30 seconds | when the grace time expires, still running jobs get 30 seconds to terminate. After that, kill signal is triggered. |
|
37
|
+
| kill_signal | SIGKILL | Signal to use kill Sidekiq process if it doesn't terminate. |
|
37
38
|
|
38
39
|
## Authors
|
39
40
|
|
@@ -1,66 +1,83 @@
|
|
1
1
|
require "get_process_mem"
|
2
2
|
require "sidekiq"
|
3
|
+
require "sidekiq/util"
|
3
4
|
|
4
5
|
module Sidekiq
|
5
6
|
# Sidekiq server middleware. Kill worker when the RSS memory exceeds limit
|
6
7
|
# after a given grace time.
|
7
8
|
class WorkerKiller
|
9
|
+
include Sidekiq::Util
|
10
|
+
|
8
11
|
MUTEX = Mutex.new
|
9
12
|
|
10
13
|
def initialize(options = {})
|
11
14
|
@max_rss = (options[:max_rss] || 0)
|
12
15
|
@grace_time = (options[:grace_time] || 15 * 60)
|
13
16
|
@shutdown_wait = (options[:shutdown_wait] || 30)
|
14
|
-
@
|
17
|
+
@kill_signal = (options[:kill_signal] || "SIGKILL")
|
15
18
|
end
|
16
19
|
|
17
|
-
def call(
|
20
|
+
def call(_worker, _job, _queue)
|
18
21
|
yield
|
19
22
|
# Skip if the max RSS is not exceeded
|
20
|
-
Sidekiq.logger.debug("current RSS is #{current_rss} MB")
|
21
23
|
return unless @max_rss > 0 && current_rss > @max_rss
|
22
|
-
#
|
23
|
-
|
24
|
+
# Launch the shutdown process
|
25
|
+
warn "current RSS #{current_rss} of #{identity} exceeds " \
|
26
|
+
"maximum RSS #{@max_rss}"
|
27
|
+
request_shutdown
|
24
28
|
end
|
25
29
|
|
26
30
|
private
|
27
31
|
|
28
|
-
def
|
32
|
+
def request_shutdown
|
29
33
|
# In another thread to allow undelying job to finish
|
30
34
|
Thread.new do
|
31
|
-
#
|
32
|
-
|
35
|
+
# Only if another thread is not already
|
36
|
+
# shutting down the Sidekiq process
|
37
|
+
shutdown if MUTEX.try_lock
|
38
|
+
end
|
39
|
+
end
|
33
40
|
|
34
|
-
|
35
|
-
|
41
|
+
def shutdown
|
42
|
+
warn "sending #{quiet_signal} to #{identity}"
|
43
|
+
signal(quiet_signal, pid)
|
36
44
|
|
37
|
-
|
38
|
-
|
39
|
-
"#{@grace_time} seconds"
|
40
|
-
sleep(@grace_time)
|
45
|
+
warn "shutting down #{identity} in #{@grace_time} seconds"
|
46
|
+
sleep(@grace_time)
|
41
47
|
|
42
|
-
|
43
|
-
|
48
|
+
warn "sending SIGTERM to #{identity}"
|
49
|
+
signal("SIGTERM", pid)
|
44
50
|
|
45
|
-
|
46
|
-
|
47
|
-
|
51
|
+
warn "waiting #{@shutdown_wait} seconds before sending " \
|
52
|
+
"#{@kill_signal} to #{identity}"
|
53
|
+
sleep(@kill_signal)
|
48
54
|
|
49
|
-
|
50
|
-
|
51
|
-
end
|
55
|
+
warn "sending #{@kill_signal} to #{identity}"
|
56
|
+
signal(@kill_signal, pid)
|
52
57
|
end
|
53
58
|
|
54
59
|
def current_rss
|
55
|
-
|
60
|
+
::GetProcessMem.new.mb
|
56
61
|
end
|
57
62
|
|
58
|
-
def
|
63
|
+
def signal(signal, pid)
|
59
64
|
::Process.kill(signal, pid)
|
60
65
|
end
|
61
66
|
|
62
67
|
def pid
|
63
|
-
|
68
|
+
::Process.pid
|
69
|
+
end
|
70
|
+
|
71
|
+
def identity
|
72
|
+
"#{hostname}:#{pid}"
|
73
|
+
end
|
74
|
+
|
75
|
+
def quiet_signal
|
76
|
+
if Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new("5.0")
|
77
|
+
"TSTP"
|
78
|
+
else
|
79
|
+
"USR1"
|
80
|
+
end
|
64
81
|
end
|
65
82
|
|
66
83
|
def warn(msg)
|
@@ -1,6 +1,11 @@
|
|
1
1
|
require "spec_helper"
|
2
2
|
|
3
3
|
describe Sidekiq::WorkerKiller do
|
4
|
+
|
5
|
+
before do
|
6
|
+
allow(subject).to receive(:warn) # silence "warn" logs
|
7
|
+
end
|
8
|
+
|
4
9
|
describe "#call" do
|
5
10
|
let(:worker){ double("worker") }
|
6
11
|
let(:job){ double("job") }
|
@@ -15,18 +20,44 @@ describe Sidekiq::WorkerKiller do
|
|
15
20
|
before do
|
16
21
|
allow(subject).to receive(:current_rss).and_return(3)
|
17
22
|
end
|
18
|
-
it "should
|
19
|
-
expect(subject).to receive(:
|
23
|
+
it "should request shutdown" do
|
24
|
+
expect(subject).to receive(:request_shutdown)
|
20
25
|
subject.call(worker, job, queue){}
|
21
26
|
end
|
22
27
|
context "but max rss is 0" do
|
23
28
|
subject{ described_class.new(max_rss: 0) }
|
24
|
-
it "should not
|
25
|
-
expect(subject).to_not receive(:
|
29
|
+
it "should not request shutdown" do
|
30
|
+
expect(subject).to_not receive(:request_shutdown)
|
26
31
|
subject.call(worker, job, queue){}
|
27
32
|
end
|
28
33
|
end
|
29
34
|
end
|
35
|
+
end
|
36
|
+
|
37
|
+
describe "#request_shutdown" do
|
38
|
+
before { allow(subject).to receive(:shutdown){ sleep 0.01 } }
|
39
|
+
it "should call shutdown" do
|
40
|
+
expect(subject).to receive(:shutdown)
|
41
|
+
subject.send(:request_shutdown).join
|
42
|
+
end
|
43
|
+
it "should not call shutdown twice when called concurrently" do
|
44
|
+
expect(subject).to receive(:shutdown).once
|
45
|
+
2.times.map{ subject.send(:request_shutdown) }.each(&:join)
|
46
|
+
end
|
47
|
+
end
|
30
48
|
|
49
|
+
describe "#quiet_signal" do
|
50
|
+
it "should give TSTP if Sidekiq version is > 5.0" do
|
51
|
+
stub_const("Sidekiq::VERSION", "5.0")
|
52
|
+
expect(subject.send :quiet_signal).to eq "TSTP"
|
53
|
+
stub_const("Sidekiq::VERSION", "5.2.1")
|
54
|
+
expect(subject.send :quiet_signal).to eq "TSTP"
|
55
|
+
end
|
56
|
+
it "should give USR1 if Sidekiq version is < 5.0" do
|
57
|
+
stub_const("Sidekiq::VERSION", "3.0")
|
58
|
+
expect(subject.send :quiet_signal).to eq "USR1"
|
59
|
+
stub_const("Sidekiq::VERSION", "4.6.7")
|
60
|
+
expect(subject.send :quiet_signal).to eq "USR1"
|
61
|
+
end
|
31
62
|
end
|
32
63
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-worker-killer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Cyrille Courtiere
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-04-
|
11
|
+
date: 2018-04-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: get_process_mem
|