resqued 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,92 @@
1
+ require "resqued/listener_proxy"
2
+ require "resqued/listener_state"
3
+
4
+ module Resqued
5
+ class ListenerPool
6
+ include Enumerable
7
+
8
+ # Public: Initialize a new pool, and store state in the given master's state.
9
+ def initialize(master_state)
10
+ @master_state = master_state
11
+ @listener_proxies = {}
12
+ # If this master is replacing an old one, there will be listeners in the state already.
13
+ @master_state.listener_states.each do |pid, ls|
14
+ @listener_proxies[pid] = ListenerProxy.new(ls)
15
+ end
16
+ end
17
+
18
+ # Public: Iterate through all active ListenerProxy instances.
19
+ def each(&block)
20
+ @listener_proxies.values.each(&block)
21
+ end
22
+
23
+ # Public: Number of active listeners.
24
+ def size
25
+ @listener_proxies.size
26
+ end
27
+
28
+ # Public: Initialize a new listener, run it, and record it as the current listener. Returns its ListenerProxy.
29
+ def start!
30
+ listener_state = ListenerState.new
31
+ listener_state.options = {
32
+ config_paths: @master_state.config_paths,
33
+ old_workers: map { |l| l.running_workers }.flatten,
34
+ listener_id: next_listener_id,
35
+ }
36
+ listener = ListenerProxy.new(listener_state)
37
+ listener.run
38
+ @master_state.listener_states[listener.pid] = listener_state
39
+ @listener_proxies[listener.pid] = listener
40
+ @master_state.current_listener_pid = listener.pid
41
+ return listener
42
+ end
43
+
44
+ # Public: Remove the given pid from the set of known listeners, and return its ListenerProxy.
45
+ def delete(pid)
46
+ @master_state.listener_states.delete(pid)
47
+ return @listener_proxies.delete(pid)
48
+ end
49
+
50
+ # Public: The current ListenerProxy, if available.
51
+ def current
52
+ @listener_proxies[current_pid]
53
+ end
54
+
55
+ # Public: The pid of the current listener, if available.
56
+ def current_pid
57
+ @master_state.current_listener_pid
58
+ end
59
+
60
+ # Public: Don't consider the current listener to be current anymore.
61
+ def clear_current!
62
+ @master_state.current_listener_pid = nil
63
+ end
64
+
65
+ # Public: Change the current listener into the last good listener.
66
+ def cycle_current
67
+ @master_state.last_good_listener_pid = @master_state.current_listener_pid
68
+ @master_state.current_listener_pid = nil
69
+ end
70
+
71
+ # Public: The last good (previous current) ListenerProxy, if available.
72
+ def last_good
73
+ @listener_proxies[last_good_pid]
74
+ end
75
+
76
+ # Public: The pid of the last good listener, if available.
77
+ def last_good_pid
78
+ @master_state.last_good_listener_pid
79
+ end
80
+
81
+ # Public: Forget which listener was the last good one.
82
+ def clear_last_good!
83
+ @master_state.last_good_listener_pid = nil
84
+ end
85
+
86
+ private
87
+
88
+ def next_listener_id
89
+ @master_state.listeners_created += 1
90
+ end
91
+ end
92
+ end
@@ -1,8 +1,8 @@
1
- require 'fcntl'
2
- require 'socket'
1
+ require "fcntl"
2
+ require "socket"
3
3
 
4
- require 'resqued/listener'
5
- require 'resqued/logging'
4
+ require "resqued/listener"
5
+ require "resqued/logging"
6
6
 
7
7
  module Resqued
8
8
  # Controls a listener process from the master process.
@@ -10,41 +10,46 @@ module Resqued
10
10
  include Resqued::Logging
11
11
 
12
12
  # Public.
13
- def initialize(options)
14
- @options = options
13
+ def initialize(state)
14
+ @state = state
15
15
  end
16
16
 
17
+ attr_reader :state
18
+
17
19
  # Public: wrap up all the things, this object is going home.
18
20
  def dispose
19
- if @master_socket
20
- @master_socket.close
21
- @master_socket = nil
21
+ if @state.master_socket
22
+ @state.master_socket.close
23
+ @state.master_socket = nil
22
24
  end
23
25
  end
24
26
 
25
27
  # Public: An IO to select on to check if there is incoming data available.
26
28
  def read_pipe
27
- @master_socket
29
+ @state.master_socket
28
30
  end
29
31
 
30
32
  # Public: The pid of the running listener process.
31
- attr_reader :pid
33
+ def pid
34
+ @state.pid
35
+ end
32
36
 
33
37
  # Public: Start the listener process.
34
38
  def run
35
39
  return if pid
40
+
36
41
  listener_socket, master_socket = UNIXSocket.pair
37
- if @pid = fork
42
+ if @state.pid = fork
38
43
  # master
39
44
  listener_socket.close
40
45
  master_socket.close_on_exec = true
41
- log "Started listener #{@pid}"
42
- @master_socket = master_socket
46
+ log "Started listener #{@state.pid}"
47
+ @state.master_socket = master_socket
43
48
  else
44
49
  # listener
45
50
  master_socket.close
46
- Master::TRAPS.each { |signal| trap(signal, 'DEFAULT') rescue nil }
47
- Listener.new(@options.merge(:socket => listener_socket)).exec
51
+ Master::TRAPS.each { |signal| trap(signal, "DEFAULT") rescue nil }
52
+ Listener.new(@state.options.merge(socket: listener_socket)).exec
48
53
  exit
49
54
  end
50
55
  end
@@ -57,46 +62,47 @@ module Resqued
57
62
 
58
63
  # Public: Get the list of workers running from this listener.
59
64
  def running_workers
60
- worker_pids.map { |pid, queue_key| { :pid => pid, :queue_key => queue_key } }
65
+ worker_pids.map { |pid, queue_key| { pid: pid, queue_key: queue_key } }
61
66
  end
62
67
 
63
68
  # Private: Map worker pids to queue names
64
69
  def worker_pids
65
- @worker_pids ||= {}
70
+ @state.worker_pids ||= {}
66
71
  end
67
72
 
68
73
  # Public: Check for updates on running worker information.
69
74
  def read_worker_status(options)
70
75
  on_activity = options[:on_activity]
71
- until @master_socket.nil?
72
- IO.select([@master_socket], nil, nil, 0) or return
73
- case line = @master_socket.readline
76
+ until @state.master_socket.nil?
77
+ IO.select([@state.master_socket], nil, nil, 0) or return
78
+ case line = @state.master_socket.readline
74
79
  when /^\+(\d+),(.*)$/
75
80
  worker_pids[$1] = $2
76
- on_activity.worker_started($1) if on_activity
81
+ on_activity&.worker_started($1)
77
82
  when /^-(\d+)$/
78
83
  worker_pids.delete($1)
79
- on_activity.worker_finished($1) if on_activity
84
+ on_activity&.worker_finished($1)
80
85
  when /^RUNNING/
81
- on_activity.listener_running(self) if on_activity
82
- when ''
86
+ on_activity&.listener_running(self)
87
+ when ""
83
88
  break
84
89
  else
85
90
  log "Malformed data from listener: #{line.inspect}"
86
91
  end
87
92
  end
88
93
  rescue EOFError, Errno::ECONNRESET
89
- @master_socket.close
90
- @master_socket = nil
94
+ @state.master_socket.close
95
+ @state.master_socket = nil
91
96
  end
92
97
 
93
98
  # Public: Tell the listener process that a worker finished.
94
99
  def worker_finished(pid)
95
- return if @master_socket.nil?
96
- @master_socket.puts(pid)
100
+ return if @state.master_socket.nil?
101
+
102
+ @state.master_socket.puts(pid)
97
103
  rescue Errno::EPIPE
98
- @master_socket.close
99
- @master_socket = nil
104
+ @state.master_socket.close
105
+ @state.master_socket = nil
100
106
  end
101
107
  end
102
108
  end
@@ -0,0 +1,8 @@
1
+ module Resqued
2
+ class ListenerState
3
+ attr_accessor :master_socket
4
+ attr_accessor :options
5
+ attr_accessor :pid
6
+ attr_accessor :worker_pids
7
+ end
8
+ end
@@ -1,4 +1,4 @@
1
- require 'mono_logger'
1
+ require "mono_logger"
2
2
 
3
3
  module Resqued
4
4
  # Mixin for any class that wants to write messages to the log file.
@@ -18,27 +18,34 @@ module Resqued
18
18
 
19
19
  class ResquedLogFormatter < ::Logger::Formatter
20
20
  def call(severity, time, progname, msg)
21
- "[%s#%6d] %5s %s -- %s\n" % [format_datetime(time), $$, severity, progname, msg2str(msg)]
21
+ sprintf "[%s#%6d] %5s %s -- %s\n",
22
+ format_datetime(time),
23
+ $$,
24
+ severity,
25
+ progname,
26
+ msg2str(msg)
22
27
  end
23
28
  end
24
29
 
25
30
  # Private: Lets our logger reopen its logfile without monologger EVEN KNOWING.
26
31
  class ResquedLoggingIOWrapper
32
+ # rubocop: disable Style/MethodMissingSuper
27
33
  def method_missing(*args)
28
34
  ::Resqued::Logging.logging_io.send(*args)
29
35
  end
36
+ # rubocop: enable Style/MethodMissingSuper
30
37
 
31
- def respond_to?(*args)
32
- ::Resqued::Logging.logging_io.respond_to?(*args)
38
+ def respond_to_missing?(method, *)
39
+ ::Resqued::Logging.logging_io.respond_to?(method)
33
40
  end
34
41
  end
35
42
 
36
43
  # Private: Get an IO to write log messages to.
37
44
  def logging_io
38
- @logging_io = nil if @logging_io && @logging_io.closed?
45
+ @logging_io = nil if @logging_io&.closed?
39
46
  @logging_io ||=
40
47
  if path = Resqued::Logging.log_file
41
- File.open(path, 'a').tap do |f|
48
+ File.open(path, "a").tap do |f|
42
49
  f.sync = true
43
50
  f.close_on_exec = true
44
51
  # Make sure we're not holding onto a stale filehandle.
@@ -60,13 +67,13 @@ module Resqued
60
67
 
61
68
  # Public.
62
69
  def log_file=(path)
63
- ENV['RESQUED_LOGFILE'] = File.expand_path(path)
70
+ ENV["RESQUED_LOGFILE"] = File.expand_path(path)
64
71
  close_log
65
72
  end
66
73
 
67
74
  # Public.
68
75
  def log_file
69
- ENV['RESQUED_LOGFILE']
76
+ ENV["RESQUED_LOGFILE"]
70
77
  end
71
78
  end
72
79
 
@@ -1,9 +1,11 @@
1
- require 'resqued/backoff'
2
- require 'resqued/listener_proxy'
3
- require 'resqued/logging'
4
- require 'resqued/pidfile'
5
- require 'resqued/procline_version'
6
- require 'resqued/sleepy'
1
+ require "resqued/backoff"
2
+ require "resqued/exec_on_hup"
3
+ require "resqued/listener_pool"
4
+ require "resqued/logging"
5
+ require "resqued/master_state"
6
+ require "resqued/pidfile"
7
+ require "resqued/procline_version"
8
+ require "resqued/sleepy"
7
9
 
8
10
  module Resqued
9
11
  # The master process.
@@ -16,19 +18,17 @@ module Resqued
16
18
  include Resqued::ProclineVersion
17
19
  include Resqued::Sleepy
18
20
 
19
- def initialize(options)
20
- @config_paths = options.fetch(:config_paths)
21
- @pidfile = options.fetch(:master_pidfile) { nil }
22
- @status_pipe = options.fetch(:status_pipe) { nil }
23
- @fast_exit = options.fetch(:fast_exit) { false }
21
+ def initialize(state, options = {})
22
+ @state = state
23
+ @status_pipe = options.fetch(:status_pipe, nil)
24
+ @listeners = ListenerPool.new(state)
24
25
  @listener_backoff = Backoff.new
25
- @listeners_created = 0
26
26
  end
27
27
 
28
28
  # Public: Starts the master process.
29
29
  def run(ready_pipe = nil)
30
30
  report_unexpected_exits
31
- with_pidfile(@pidfile) do
31
+ with_pidfile(@state.pidfile) do
32
32
  write_procline
33
33
  install_signal_handlers
34
34
  if ready_pipe
@@ -42,32 +42,39 @@ module Resqued
42
42
 
43
43
  # Private: dat main loop.
44
44
  def go_ham
45
+ # If we're resuming, we'll want to recycle the existing listener now.
46
+ prepare_new_listener
47
+
45
48
  loop do
46
49
  read_listeners
47
50
  reap_all_listeners(Process::WNOHANG)
48
- start_listener unless @paused
51
+ start_listener unless @state.paused
49
52
  case signal = SIGNAL_QUEUE.shift
50
53
  when nil
51
54
  yawn(@listener_backoff.how_long? || 30.0)
52
55
  when :INFO
53
56
  dump_object_counts
54
57
  when :HUP
58
+ if @state.exec_on_hup
59
+ log "Execing a new master"
60
+ ExecOnHUP.exec!(@state)
61
+ end
55
62
  reopen_logs
56
63
  log "Restarting listener with new configuration and application."
57
64
  prepare_new_listener
58
65
  when :USR2
59
66
  log "Pause job processing"
60
- @paused = true
61
- kill_listener(:QUIT, @current_listener)
62
- @current_listener = nil
67
+ @state.paused = true
68
+ kill_listener(:QUIT, @listeners.current)
69
+ @listeners.clear_current!
63
70
  when :CONT
64
71
  log "Resume job processing"
65
- @paused = false
72
+ @state.paused = false
66
73
  kill_all_listeners(:CONT)
67
74
  when :INT, :TERM, :QUIT
68
75
  log "Shutting down..."
69
76
  kill_all_listeners(signal)
70
- wait_for_workers unless @fast_exit
77
+ wait_for_workers unless @state.fast_exit
71
78
  break
72
79
  end
73
80
  end
@@ -85,14 +92,14 @@ module Resqued
85
92
  end
86
93
  top = 10
87
94
  log "#{total} objects. top #{top}:"
88
- counts.sort_by { |name, count| count }.reverse.each_with_index do |(name, count), i|
89
- if i < top
90
- diff = ""
91
- if last = @last_counts && @last_counts[name]
92
- diff = " (#{'%+d' % (count - last)})"
93
- end
94
- log " #{count} #{name}#{diff}"
95
+ counts.sort_by { |_, count| -count }.each_with_index do |(name, count), i|
96
+ next unless i < top
97
+
98
+ diff = ""
99
+ if last = @last_counts && @last_counts[name]
100
+ diff = sprintf(" (%+d)", (count - last))
95
101
  end
102
+ log " #{count} #{name}#{diff}"
96
103
  end
97
104
  @last_counts = counts
98
105
  log GC.stat.inspect
@@ -100,59 +107,45 @@ module Resqued
100
107
  log "Error while counting objects: #{e}"
101
108
  end
102
109
 
103
- # Private: Map listener pids to ListenerProxy objects.
104
- def listener_pids
105
- @listener_pids ||= {}
106
- end
107
-
108
- # Private: All the ListenerProxy objects.
109
- def all_listeners
110
- listener_pids.values
111
- end
112
-
113
110
  def start_listener
114
- return if @current_listener || @listener_backoff.wait?
115
- @current_listener = ListenerProxy.new(:config_paths => @config_paths, :old_workers => all_listeners.map { |l| l.running_workers }.flatten, :listener_id => next_listener_id)
116
- @current_listener.run
117
- listener_status @current_listener, 'start'
111
+ return if @listeners.current || @listener_backoff.wait?
112
+
113
+ listener = @listeners.start!
114
+ listener_status listener, "start"
118
115
  @listener_backoff.started
119
- listener_pids[@current_listener.pid] = @current_listener
120
116
  write_procline
121
117
  end
122
118
 
123
- def next_listener_id
124
- @listeners_created += 1
125
- end
126
-
127
119
  def read_listeners
128
- all_listeners.each do |l|
129
- l.read_worker_status(:on_activity => self)
120
+ @listeners.each do |l|
121
+ l.read_worker_status(on_activity: self)
130
122
  end
131
123
  end
132
124
 
133
125
  # Listener message: A worker just started working.
134
126
  def worker_started(pid)
135
- worker_status(pid, 'start')
127
+ worker_status(pid, "start")
136
128
  end
137
129
 
138
130
  # Listener message: A worker just stopped working.
139
131
  #
140
132
  # Forwards the message to the other listeners.
141
133
  def worker_finished(pid)
142
- worker_status(pid, 'stop')
143
- all_listeners.each do |other|
134
+ worker_status(pid, "stop")
135
+ @listeners.each do |other|
144
136
  other.worker_finished(pid)
145
137
  end
146
138
  end
147
139
 
148
- # Listener message: A listener finished booting, and is ready to start workers.
140
+ # Listener message: A listener finished booting, and is ready to
141
+ # start workers.
149
142
  #
150
143
  # Promotes a booting listener to be the current listener.
151
144
  def listener_running(listener)
152
- listener_status(listener, 'ready')
153
- if listener == @current_listener
154
- kill_listener(:QUIT, @last_good_listener)
155
- @last_good_listener = nil
145
+ listener_status(listener, "ready")
146
+ if listener == @listeners.current
147
+ kill_listener(:QUIT, @listeners.last_good)
148
+ @listeners.clear_last_good!
156
149
  else
157
150
  # This listener didn't receive the last SIGQUIT we sent.
158
151
  # (It was probably sent before the listener had set up its traps.)
@@ -165,23 +158,26 @@ module Resqued
165
158
  #
166
159
  # The old one will be killed when the new one is ready for workers.
167
160
  def prepare_new_listener
168
- if @last_good_listener
169
- # The last_good_listener is still running because we got another HUP before the new listener finished booting.
170
- # Keep the last_good_listener (where all the workers are) and kill the booting current_listener. We'll start a new one.
171
- kill_listener(:QUIT, @current_listener)
161
+ if @listeners.last_good
162
+ # The last good listener is still running because we got another
163
+ # HUP before the new listener finished booting.
164
+ # Keep the last_good_listener (where all the workers are) and
165
+ # kill the booting current_listener. We'll start a new one.
166
+ kill_listener(:QUIT, @listeners.current)
167
+ # Indicate to `start_listener` that it should start a new
168
+ # listener.
169
+ @listeners.clear_current!
172
170
  else
173
- @last_good_listener = @current_listener
171
+ @listeners.cycle_current
174
172
  end
175
- # Indicate to `start_listener` that it should start a new listener.
176
- @current_listener = nil
177
173
  end
178
174
 
179
175
  def kill_listener(signal, listener)
180
- listener.kill(signal) if listener
176
+ listener&.kill(signal)
181
177
  end
182
178
 
183
179
  def kill_all_listeners(signal)
184
- all_listeners.each do |l|
180
+ @listeners.each do |l|
185
181
  l.kill(signal)
186
182
  end
187
183
  end
@@ -191,80 +187,78 @@ module Resqued
191
187
  end
192
188
 
193
189
  def reap_all_listeners(waitpid_flags = 0)
194
- begin
195
- lpid, status = Process.waitpid2(-1, waitpid_flags)
196
- if lpid
190
+ loop do
191
+ begin
192
+ lpid, status = Process.waitpid2(-1, waitpid_flags)
193
+ return unless lpid
194
+
197
195
  log "Listener exited #{status}"
198
- if @current_listener && @current_listener.pid == lpid
196
+
197
+ if @listeners.current_pid == lpid
199
198
  @listener_backoff.died
200
- @current_listener = nil
199
+ @listeners.clear_current!
201
200
  end
202
201
 
203
- if @last_good_listener && @last_good_listener.pid == lpid
204
- @last_good_listener = nil
202
+ if @listeners.last_good_pid == lpid
203
+ @state.clear_last_good!
205
204
  end
206
- dead_listener = listener_pids.delete(lpid)
207
- listener_status dead_listener, 'stop'
205
+
206
+ dead_listener = @listeners.delete(lpid)
207
+ listener_status dead_listener, "stop"
208
208
  dead_listener.dispose
209
209
  write_procline
210
- else
210
+ rescue Errno::ECHILD
211
211
  return
212
212
  end
213
- rescue Errno::ECHILD
214
- return
215
- end while true
213
+ end
216
214
  end
217
215
 
218
- SIGNALS = [ :HUP, :INT, :USR2, :CONT, :TERM, :QUIT ]
219
- OPTIONAL_SIGNALS = [ :INFO ]
220
- OTHER_SIGNALS = [:CHLD, 'EXIT']
216
+ SIGNALS = [:HUP, :INT, :USR2, :CONT, :TERM, :QUIT].freeze
217
+ OPTIONAL_SIGNALS = [:INFO].freeze
218
+ OTHER_SIGNALS = [:CHLD, "EXIT"].freeze
221
219
  TRAPS = SIGNALS + OPTIONAL_SIGNALS + OTHER_SIGNALS
222
220
 
223
- SIGNAL_QUEUE = []
221
+ SIGNAL_QUEUE = [] # rubocop: disable Style/MutableConstant
224
222
 
225
223
  def install_signal_handlers
226
224
  trap(:CHLD) { awake }
227
- SIGNALS.each { |signal| trap(signal) { SIGNAL_QUEUE << signal ; awake } }
228
- OPTIONAL_SIGNALS.each { |signal| trap(signal) { SIGNAL_QUEUE << signal ; awake } rescue nil }
225
+ SIGNALS.each { |signal| trap(signal) { SIGNAL_QUEUE << signal; awake } }
226
+ OPTIONAL_SIGNALS.each { |signal| trap(signal) { SIGNAL_QUEUE << signal; awake } rescue nil }
229
227
  end
230
228
 
231
229
  def report_unexpected_exits
232
- trap('EXIT') do
230
+ trap("EXIT") do
233
231
  log("EXIT #{$!.inspect}")
234
- if $!
235
- $!.backtrace.each do |line|
236
- log(line)
237
- end
232
+ $!&.backtrace&.each do |line|
233
+ log(line)
238
234
  end
239
235
  end
240
236
  end
241
237
 
242
238
  def no_more_unexpected_exits
243
- trap('EXIT', 'DEFAULT')
239
+ trap("EXIT", "DEFAULT")
244
240
  end
245
241
 
246
242
  def yawn(duration)
247
- super(duration, all_listeners.map { |l| l.read_pipe })
243
+ super(duration, @listeners.map { |l| l.read_pipe })
248
244
  end
249
245
 
250
246
  def write_procline
251
- $0 = "#{procline_version} master [gen #{@listeners_created}] [#{listener_pids.size} running] #{ARGV.join(' ')}"
247
+ $0 = "#{procline_version} master [gen #{@state.listeners_created}] [#{@listeners.size} running] #{ARGV.join(' ')}"
252
248
  end
253
249
 
254
250
  def listener_status(listener, status)
255
- if listener && listener.pid
256
- status_message('listener', listener.pid, status)
251
+ if listener&.pid
252
+ status_message("listener", listener.pid, status)
257
253
  end
258
254
  end
259
255
 
260
256
  def worker_status(pid, status)
261
- status_message('worker', pid, status)
257
+ status_message("worker", pid, status)
262
258
  end
263
259
 
264
260
  def status_message(type, pid, status)
265
- if @status_pipe
266
- @status_pipe.write("#{type},#{pid},#{status}\n")
267
- end
261
+ @status_pipe&.write("#{type},#{pid},#{status}\n")
268
262
  end
269
263
  end
270
264
  end