resqued 0.9.0 → 0.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CHANGES.md +23 -0
- data/docs/signals.md +4 -0
- data/exe/resqued +41 -22
- data/lib/resqued.rb +5 -5
- data/lib/resqued/config.rb +7 -7
- data/lib/resqued/config/after_fork.rb +1 -1
- data/lib/resqued/config/base.rb +1 -1
- data/lib/resqued/config/before_fork.rb +1 -1
- data/lib/resqued/config/worker.rb +13 -13
- data/lib/resqued/daemon.rb +1 -0
- data/lib/resqued/exec_on_hup.rb +43 -0
- data/lib/resqued/listener.rb +51 -48
- data/lib/resqued/listener_pool.rb +97 -0
- data/lib/resqued/listener_proxy.rb +40 -31
- data/lib/resqued/listener_state.rb +8 -0
- data/lib/resqued/logging.rb +15 -8
- data/lib/resqued/master.rb +94 -98
- data/lib/resqued/master_state.rb +73 -0
- data/lib/resqued/procline_version.rb +2 -2
- data/lib/resqued/sleepy.rb +6 -4
- data/lib/resqued/test_case.rb +3 -3
- data/lib/resqued/version.rb +1 -1
- data/lib/resqued/worker.rb +21 -14
- data/spec/fixtures/test_case_after_fork_raises.rb +5 -2
- data/spec/fixtures/test_case_before_fork_raises.rb +4 -1
- data/spec/fixtures/test_case_environment.rb +3 -1
- data/spec/integration/listener_still_starting_spec.rb +24 -0
- data/spec/integration/master_inherits_child_spec.rb +85 -0
- data/spec/integration/restart_spec.rb +21 -0
- data/spec/resqued/backoff_spec.rb +27 -27
- data/spec/resqued/config/fork_event_spec.rb +8 -8
- data/spec/resqued/config/worker_spec.rb +68 -55
- data/spec/resqued/config_spec.rb +6 -6
- data/spec/resqued/sleepy_spec.rb +10 -11
- data/spec/resqued/test_case_spec.rb +7 -7
- data/spec/spec_helper.rb +6 -1
- data/spec/support/custom_matchers.rb +10 -2
- data/spec/support/extra-child-shim +6 -0
- data/spec/support/resqued_integration_helpers.rb +50 -0
- data/spec/support/resqued_path.rb +11 -0
- metadata +57 -36
- data/exe/resqued-listener +0 -6
@@ -0,0 +1,73 @@
|
|
1
|
+
module Resqued
|
2
|
+
class MasterState
|
3
|
+
def initialize
|
4
|
+
@listeners_created = 0
|
5
|
+
@listener_states = {}
|
6
|
+
end
|
7
|
+
|
8
|
+
# Public: When starting fresh, from command-line options, assign the initial values.
|
9
|
+
def init(options)
|
10
|
+
@config_paths = options.fetch(:config_paths)
|
11
|
+
@exec_on_hup = options.fetch(:exec_on_hup, false)
|
12
|
+
@fast_exit = options.fetch(:fast_exit, false)
|
13
|
+
@pidfile = options.fetch(:master_pidfile, nil)
|
14
|
+
end
|
15
|
+
|
16
|
+
# Public: Restore state from a serialized form.
|
17
|
+
def restore(data)
|
18
|
+
@config_paths = data[:config_paths]
|
19
|
+
@current_listener_pid = data[:current_listener_pid]
|
20
|
+
@exec_on_hup = data[:exec_on_hup]
|
21
|
+
@fast_exit = data[:fast_exit]
|
22
|
+
@last_good_listener_pid = data[:last_good_listener_pid]
|
23
|
+
@listeners_created = data[:listeners_created]
|
24
|
+
data[:listener_states].each do |lsh|
|
25
|
+
@listener_states[lsh[:pid]] = ListenerState.new.tap do |ls|
|
26
|
+
ls.master_socket = lsh[:master_socket] && Socket.for_fd(lsh[:master_socket])
|
27
|
+
ls.options = lsh[:options]
|
28
|
+
ls.pid = lsh[:pid]
|
29
|
+
ls.worker_pids = lsh[:worker_pids]
|
30
|
+
end
|
31
|
+
end
|
32
|
+
@paused = data[:paused]
|
33
|
+
@pidfile = data[:pidfile]
|
34
|
+
end
|
35
|
+
|
36
|
+
# Public: Return this state so that it can be serialized.
|
37
|
+
def to_h
|
38
|
+
{
|
39
|
+
config_paths: @config_paths,
|
40
|
+
current_listener_pid: @current_listener_pid,
|
41
|
+
exec_on_hup: @exec_on_hup,
|
42
|
+
fast_exit: @fast_exit,
|
43
|
+
last_good_listener_pid: @last_good_listener_pid,
|
44
|
+
listeners_created: @listeners_created,
|
45
|
+
listener_states: @listener_states.values.map { |ls|
|
46
|
+
{
|
47
|
+
master_socket: ls.master_socket&.to_i,
|
48
|
+
options: ls.options,
|
49
|
+
pid: ls.pid,
|
50
|
+
worker_pids: ls.worker_pids,
|
51
|
+
}
|
52
|
+
},
|
53
|
+
paused: @paused,
|
54
|
+
pidfile: @pidfile,
|
55
|
+
}
|
56
|
+
end
|
57
|
+
|
58
|
+
# Public: Return an array of open sockets or other file handles that should be forwarded to a new master.
|
59
|
+
def sockets
|
60
|
+
@listener_states.values.map { |l| l.master_socket }.compact
|
61
|
+
end
|
62
|
+
|
63
|
+
attr_reader :config_paths
|
64
|
+
attr_accessor :current_listener_pid
|
65
|
+
attr_reader :exec_on_hup
|
66
|
+
attr_reader :fast_exit
|
67
|
+
attr_accessor :last_good_listener_pid
|
68
|
+
attr_accessor :listeners_created
|
69
|
+
attr_reader :listener_states
|
70
|
+
attr_accessor :paused
|
71
|
+
attr_reader :pidfile
|
72
|
+
end
|
73
|
+
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require "resqued/version"
|
2
2
|
|
3
3
|
module Resqued
|
4
4
|
module ProclineVersion
|
@@ -6,7 +6,7 @@ module Resqued
|
|
6
6
|
@version ||=
|
7
7
|
begin
|
8
8
|
# If we've built a custom version, this should show the custom version.
|
9
|
-
Gem.loaded_specs[
|
9
|
+
Gem.loaded_specs["resqued"].version.to_s
|
10
10
|
rescue Object
|
11
11
|
# If this isn't a gem, fall back to the version in resqued/version.rb.
|
12
12
|
Resqued::VERSION
|
data/lib/resqued/sleepy.rb
CHANGED
@@ -1,9 +1,11 @@
|
|
1
|
-
require
|
2
|
-
require
|
1
|
+
require "fcntl"
|
2
|
+
require "kgio"
|
3
3
|
|
4
4
|
module Resqued
|
5
5
|
module Sleepy
|
6
|
-
# Public: Like sleep, but the sleep is interrupted if input is
|
6
|
+
# Public: Like sleep, but the sleep is interrupted if input is
|
7
|
+
# detected on one of the provided IO objects, or if `awake` is
|
8
|
+
# called (e.g. from a signal handler).
|
7
9
|
def yawn(duration, *inputs)
|
8
10
|
if duration > 0
|
9
11
|
inputs = [self_pipe[0]] + [inputs].flatten.compact
|
@@ -14,7 +16,7 @@ module Resqued
|
|
14
16
|
|
15
17
|
# Public: Break out of `yawn`.
|
16
18
|
def awake
|
17
|
-
self_pipe[1].kgio_trywrite(
|
19
|
+
self_pipe[1].kgio_trywrite(".")
|
18
20
|
end
|
19
21
|
|
20
22
|
# Private.
|
data/lib/resqued/test_case.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
|
-
require
|
2
|
-
require
|
1
|
+
require "resqued/config"
|
2
|
+
require "resqued/runtime_info"
|
3
3
|
|
4
4
|
module Resqued
|
5
5
|
module TestCase
|
@@ -17,7 +17,7 @@ module Resqued
|
|
17
17
|
config = Resqued::Config.new(paths)
|
18
18
|
config.before_fork(RuntimeInfo.new)
|
19
19
|
config.build_workers
|
20
|
-
config.after_fork(Resque::Worker.new(
|
20
|
+
config.after_fork(Resque::Worker.new("*"))
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
data/lib/resqued/version.rb
CHANGED
data/lib/resqued/worker.rb
CHANGED
@@ -1,17 +1,17 @@
|
|
1
|
-
require
|
2
|
-
require
|
1
|
+
require "resque"
|
2
|
+
require "digest"
|
3
3
|
|
4
|
-
require
|
5
|
-
require
|
4
|
+
require "resqued/backoff"
|
5
|
+
require "resqued/logging"
|
6
6
|
|
7
7
|
module Resqued
|
8
8
|
# Models a worker process.
|
9
9
|
class Worker
|
10
10
|
include Resqued::Logging
|
11
11
|
|
12
|
-
DEFAULT_WORKER_FACTORY =
|
12
|
+
DEFAULT_WORKER_FACTORY = lambda { |queues|
|
13
13
|
resque_worker = Resque::Worker.new(*queues)
|
14
|
-
resque_worker.term_child = true if resque_worker.respond_to?(
|
14
|
+
resque_worker.term_child = true if resque_worker.respond_to?("term_child=")
|
15
15
|
redis_client = Resque.redis.respond_to?(:_client) ? Resque.redis._client : Resque.redis.client
|
16
16
|
redis_client.reconnect
|
17
17
|
resque_worker
|
@@ -44,12 +44,13 @@ module Resqued
|
|
44
44
|
|
45
45
|
# Public: A string that compares if this worker is equivalent to a worker in another Resqued::Listener.
|
46
46
|
def queue_key
|
47
|
-
Digest::SHA256.hexdigest(queues.sort.join(
|
47
|
+
Digest::SHA256.hexdigest(queues.sort.join(";"))
|
48
48
|
end
|
49
49
|
|
50
50
|
# Public: Claim this worker for another listener's worker.
|
51
51
|
def wait_for(pid)
|
52
52
|
raise "Already running #{@pid} (can't wait for #{pid})" if @pid
|
53
|
+
|
53
54
|
@self_started = false
|
54
55
|
@pids << pid
|
55
56
|
@pid = pid
|
@@ -57,16 +58,17 @@ module Resqued
|
|
57
58
|
|
58
59
|
# Public: The old worker process finished!
|
59
60
|
def finished!(process_status)
|
60
|
-
|
61
|
-
|
61
|
+
summary = "(#{@pid}/#{@pids.inspect}/self_started=#{@self_started}/killed=#{@killed})"
|
62
|
+
if process_status.nil? && !@self_started
|
63
|
+
log :debug, "#{summary} I am no longer blocked."
|
62
64
|
@pid = nil
|
63
65
|
@backoff.died unless @killed
|
64
|
-
elsif !
|
65
|
-
log :debug, "
|
66
|
+
elsif !process_status.nil? && @self_started
|
67
|
+
log :debug, "#{summary} I exited: #{process_status}"
|
66
68
|
@pid = nil
|
67
69
|
@backoff.died unless @killed
|
68
70
|
else
|
69
|
-
log :debug, "
|
71
|
+
log :debug, "#{summary} Reports of my death are highly exaggerated (#{process_status.inspect})"
|
70
72
|
end
|
71
73
|
end
|
72
74
|
|
@@ -78,6 +80,7 @@ module Resqued
|
|
78
80
|
# Public: Start a job, if there's one waiting in one of my queues.
|
79
81
|
def try_start
|
80
82
|
return if @backoff.wait?
|
83
|
+
|
81
84
|
@backoff.started
|
82
85
|
@self_started = true
|
83
86
|
@killed = false
|
@@ -87,8 +90,12 @@ module Resqued
|
|
87
90
|
log "Forked worker #{@pid}"
|
88
91
|
else
|
89
92
|
# In case we get a signal before resque is ready for it.
|
90
|
-
Resqued::Listener::ALL_SIGNALS.each { |signal| trap(signal,
|
91
|
-
|
93
|
+
Resqued::Listener::ALL_SIGNALS.each { |signal| trap(signal, "DEFAULT") }
|
94
|
+
# Continue ignoring SIGHUP, though.
|
95
|
+
trap(:HUP) {}
|
96
|
+
# If we get a QUIT during boot, just spin back down.
|
97
|
+
trap(:QUIT) { exit! 0 }
|
98
|
+
|
92
99
|
$0 = "STARTING RESQUE FOR #{queues.join(',')}"
|
93
100
|
resque_worker = @worker_factory.call(queues)
|
94
101
|
@config.after_fork(resque_worker)
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe "Listener still starting on SIGHUP" do
|
4
|
+
include ResquedIntegrationHelpers
|
5
|
+
|
6
|
+
it "expect master not to crash" do
|
7
|
+
start_resqued config: <<-CONFIG
|
8
|
+
before_fork do
|
9
|
+
sleep 1
|
10
|
+
end
|
11
|
+
CONFIG
|
12
|
+
expect_running listener: "listener #1"
|
13
|
+
restart_resqued
|
14
|
+
sleep 2
|
15
|
+
expect_running listener: "listener #2"
|
16
|
+
end
|
17
|
+
|
18
|
+
after do
|
19
|
+
begin
|
20
|
+
Process.kill(:QUIT, @pid) if @pid
|
21
|
+
rescue Errno::ESRCH
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "timeout"
|
3
|
+
|
4
|
+
describe "Resqued master with an extra child process" do
|
5
|
+
include ResquedPath
|
6
|
+
|
7
|
+
# Starts resqued with an extra child process.
|
8
|
+
def start_resqued_with_extra_child
|
9
|
+
shim_path = File.expand_path("../support/extra-child-shim", File.dirname(__FILE__))
|
10
|
+
|
11
|
+
config_path = File.join(SPEC_TEMPDIR, "config.rb")
|
12
|
+
File.write(config_path, <<-CONFIG)
|
13
|
+
before_fork { File.write(ENV["LISTENER_PIDFILE"], $$.to_s) }
|
14
|
+
CONFIG
|
15
|
+
|
16
|
+
logfile = File.join(SPEC_TEMPDIR, "resqued.log")
|
17
|
+
File.write(logfile, "") # truncate it
|
18
|
+
|
19
|
+
env = {
|
20
|
+
"LISTENER_PIDFILE" => listener_pidfile,
|
21
|
+
"EXTRA_CHILD_PIDFILE" => extra_child_pidfile,
|
22
|
+
}
|
23
|
+
|
24
|
+
pid = spawn(env, shim_path, resqued_path, "--logfile", logfile, config_path)
|
25
|
+
sleep 2.0
|
26
|
+
pid
|
27
|
+
end
|
28
|
+
|
29
|
+
let(:extra_child_pidfile) { File.join(SPEC_TEMPDIR, "extra-child.pid") }
|
30
|
+
def extra_child_pid
|
31
|
+
File.read(extra_child_pidfile).to_i
|
32
|
+
end
|
33
|
+
|
34
|
+
let(:listener_pidfile) { File.join(File.join(SPEC_TEMPDIR, "listener.pid")) }
|
35
|
+
def listener_pid
|
36
|
+
File.read(listener_pidfile).to_i
|
37
|
+
end
|
38
|
+
|
39
|
+
before do
|
40
|
+
File.unlink(extra_child_pidfile) rescue nil
|
41
|
+
File.unlink(listener_pidfile) rescue nil
|
42
|
+
@resqued_pid = start_resqued_with_extra_child
|
43
|
+
end
|
44
|
+
|
45
|
+
after do
|
46
|
+
kill_safely(:TERM) { @resqued_pid }
|
47
|
+
kill_safely(:KILL) { extra_child_pid }
|
48
|
+
sleep 0.1
|
49
|
+
kill_safely(:KILL) { @resqued_pid }
|
50
|
+
end
|
51
|
+
|
52
|
+
it "doesn't exit when listener dies unexpectedly" do
|
53
|
+
# Kill off the listener process.
|
54
|
+
first_listener_pid = listener_pid
|
55
|
+
Process.kill :QUIT, first_listener_pid
|
56
|
+
# Let Resqued::Backoff decide it's OK to start the listener again.
|
57
|
+
sleep 2.5
|
58
|
+
# Resqued should start a new listener to replace the dead one.
|
59
|
+
expect(listener_pid).not_to eq(first_listener_pid)
|
60
|
+
end
|
61
|
+
|
62
|
+
it "exits when listeners have all exited during shutdown" do
|
63
|
+
# Do a normal shutdown.
|
64
|
+
Process.kill :QUIT, @resqued_pid
|
65
|
+
# Expect the resqued process to exit.
|
66
|
+
expect(Timeout.timeout(5.0) { Process.waitpid(@resqued_pid) }).to eq(@resqued_pid)
|
67
|
+
end
|
68
|
+
|
69
|
+
it "doesn't crash when extra child exits" do
|
70
|
+
# Kill off the extra child process. Resqued should wait on it, but not exit.
|
71
|
+
Process.kill :KILL, extra_child_pid
|
72
|
+
sleep 1.0
|
73
|
+
# The resqued process should not have exited.
|
74
|
+
expect(Process.waitpid(@resqued_pid, Process::WNOHANG)).to be_nil
|
75
|
+
expect(Process.kill(0, @resqued_pid)).to eq(1)
|
76
|
+
end
|
77
|
+
|
78
|
+
def kill_safely(signal)
|
79
|
+
return unless pid = yield
|
80
|
+
|
81
|
+
Process.kill(signal, pid)
|
82
|
+
rescue Errno::ESRCH, Errno::ENOENT
|
83
|
+
# Process isn't there anymore, or pidfile isn't there. :+1:
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
describe "Resqued can restart" do
|
4
|
+
include ResquedIntegrationHelpers
|
5
|
+
|
6
|
+
it "expect to be able to restart" do
|
7
|
+
start_resqued
|
8
|
+
expect_running listener: "listener #1"
|
9
|
+
restart_resqued
|
10
|
+
expect_running listener: "listener #2"
|
11
|
+
stop_resqued
|
12
|
+
expect_not_running
|
13
|
+
end
|
14
|
+
|
15
|
+
after do
|
16
|
+
begin
|
17
|
+
Process.kill(:QUIT, @pid) if @pid
|
18
|
+
rescue Errno::ESRCH
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -1,61 +1,61 @@
|
|
1
|
-
require
|
1
|
+
require "spec_helper"
|
2
2
|
|
3
|
-
require
|
3
|
+
require "resqued/backoff"
|
4
4
|
|
5
5
|
describe Resqued::Backoff do
|
6
|
-
let(:backoff) { described_class.new(:
|
6
|
+
let(:backoff) { described_class.new(min: 0.5, max: 64.0) }
|
7
7
|
|
8
|
-
it
|
9
|
-
expect(backoff.wait?).to
|
8
|
+
it "can start on the first try" do
|
9
|
+
expect(backoff.wait?).to be_falsey
|
10
10
|
end
|
11
11
|
|
12
|
-
it
|
12
|
+
it "has no waiting at first" do
|
13
13
|
expect(backoff.how_long?).to be_nil
|
14
14
|
end
|
15
15
|
|
16
|
-
context
|
16
|
+
context "after expected exits" do
|
17
17
|
before { 3.times { backoff.started } }
|
18
|
-
it { expect(backoff.wait?).to
|
18
|
+
it { expect(backoff.wait?).to be true }
|
19
19
|
it { expect(backoff.how_long?).to be_close_to(0.5) }
|
20
20
|
end
|
21
21
|
|
22
|
-
context
|
23
|
-
before { 1.times { backoff.started
|
24
|
-
it { expect(backoff.wait?).to
|
22
|
+
context "after one quick exit" do
|
23
|
+
before { 1.times { backoff.started; backoff.died } }
|
24
|
+
it { expect(backoff.wait?).to be true }
|
25
25
|
it { expect(backoff.how_long?).to be_close_to(1.0) }
|
26
26
|
end
|
27
27
|
|
28
|
-
context
|
29
|
-
before { 2.times { backoff.started
|
30
|
-
it { expect(backoff.wait?).to
|
28
|
+
context "after two quick starts" do
|
29
|
+
before { 2.times { backoff.started; backoff.died } }
|
30
|
+
it { expect(backoff.wait?).to be true }
|
31
31
|
it { expect(backoff.how_long?).to be_close_to(2.0) }
|
32
32
|
end
|
33
33
|
|
34
|
-
context
|
35
|
-
before { 6.times { backoff.started
|
36
|
-
it { expect(backoff.wait?).to
|
34
|
+
context "after five quick starts" do
|
35
|
+
before { 6.times { backoff.started; backoff.died } }
|
36
|
+
it { expect(backoff.wait?).to be true }
|
37
37
|
it { expect(backoff.how_long?).to be_close_to(32.0) }
|
38
38
|
end
|
39
39
|
|
40
|
-
context
|
41
|
-
before { 7.times { backoff.started
|
42
|
-
it { expect(backoff.wait?).to
|
40
|
+
context "after six quick starts" do
|
41
|
+
before { 7.times { backoff.started; backoff.died } }
|
42
|
+
it { expect(backoff.wait?).to be true }
|
43
43
|
it { expect(backoff.how_long?).to be_close_to(64.0) }
|
44
44
|
end
|
45
45
|
|
46
|
-
context
|
47
|
-
before { 8.times { backoff.started
|
48
|
-
it { expect(backoff.wait?).to
|
46
|
+
context "does not wait longer than 64s" do
|
47
|
+
before { 8.times { backoff.started; backoff.died } }
|
48
|
+
it { expect(backoff.wait?).to be true }
|
49
49
|
it { expect(backoff.how_long?).to be_close_to(64.0) }
|
50
|
-
it
|
50
|
+
it "and resets after an expected exit" do
|
51
51
|
backoff.started
|
52
52
|
backoff.started
|
53
|
-
expect(backoff.wait?).to
|
53
|
+
expect(backoff.wait?).to be true
|
54
54
|
expect(backoff.how_long?).to be_close_to(0.5)
|
55
55
|
end
|
56
56
|
end
|
57
57
|
|
58
|
-
def be_close_to(
|
59
|
-
be_within(0.005).of(
|
58
|
+
def be_close_to(number)
|
59
|
+
be_within(0.005).of(number)
|
60
60
|
end
|
61
61
|
end
|