resqued 0.8.5 → 0.10.2

data/lib/resqued/listener_pool.rb

@@ -0,0 +1,97 @@
+require "resqued/listener_proxy"
+require "resqued/listener_state"
+
+module Resqued
+  class ListenerPool
+    include Enumerable
+
+    # Public: Initialize a new pool, and store state in the given master's state.
+    def initialize(master_state)
+      @master_state = master_state
+      @listener_proxies = {}
+      # If this master is replacing an old one, there will be listeners in the state already.
+      @master_state.listener_states.each do |pid, ls|
+        @listener_proxies[pid] = ListenerProxy.new(ls)
+      end
+    end
+
+    # Public: Iterate through all active ListenerProxy instances.
+    def each(&block)
+      @listener_proxies.values.each(&block)
+    end
+
+    # Public: Number of active listeners.
+    def size
+      @listener_proxies.size
+    end
+
+    # Public: Are the listeners all gone?
+    def empty?
+      @listener_proxies.empty?
+    end
+
+    # Public: Initialize a new listener, run it, and record it as the current listener. Returns its ListenerProxy.
+    def start!
+      listener_state = ListenerState.new
+      listener_state.options = {
+        config_paths: @master_state.config_paths,
+        old_workers: map { |l| l.running_workers }.flatten,
+        listener_id: next_listener_id,
+      }
+      listener = ListenerProxy.new(listener_state)
+      listener.run
+      @master_state.listener_states[listener.pid] = listener_state
+      @listener_proxies[listener.pid] = listener
+      @master_state.current_listener_pid = listener.pid
+      return listener
+    end
+
+    # Public: Remove the given pid from the set of known listeners, and return its ListenerProxy.
+    def delete(pid)
+      @master_state.listener_states.delete(pid)
+      return @listener_proxies.delete(pid)
+    end
+
+    # Public: The current ListenerProxy, if available.
+    def current
+      @listener_proxies[current_pid]
+    end
+
+    # Public: The pid of the current listener, if available.
+    def current_pid
+      @master_state.current_listener_pid
+    end
+
+    # Public: Don't consider the current listener to be current anymore.
+    def clear_current!
+      @master_state.current_listener_pid = nil
+    end
+
+    # Public: Change the current listener into the last good listener.
+    def cycle_current
+      @master_state.last_good_listener_pid = @master_state.current_listener_pid
+      @master_state.current_listener_pid = nil
+    end
+
+    # Public: The last good (previous current) ListenerProxy, if available.
+    def last_good
+      @listener_proxies[last_good_pid]
+    end
+
+    # Public: The pid of the last good listener, if available.
+    def last_good_pid
+      @master_state.last_good_listener_pid
+    end
+
+    # Public: Forget which listener was the last good one.
+    def clear_last_good!
+      @master_state.last_good_listener_pid = nil
+    end
+
+    private
+
+    def next_listener_id
+      @master_state.listeners_created += 1
+    end
+  end
+end

data/lib/resqued/listener_proxy.rb

@@ -1,8 +1,8 @@
-require 'fcntl'
-require 'socket'
+require "fcntl"
+require "socket"
 
-require 'resqued/listener'
-require 'resqued/logging'
+require "resqued/listener"
+require "resqued/logging"
 
 module Resqued
   # Controls a listener process from the master process.
@@ -10,41 +10,46 @@ module Resqued
     include Resqued::Logging
 
     # Public.
-    def initialize(options)
-      @options = options
+    def initialize(state)
+      @state = state
     end
 
+    attr_reader :state
+
     # Public: wrap up all the things, this object is going home.
     def dispose
-      if @master_socket
-        @master_socket.close
-        @master_socket = nil
+      if @state.master_socket
+        @state.master_socket.close
+        @state.master_socket = nil
       end
     end
 
     # Public: An IO to select on to check if there is incoming data available.
     def read_pipe
-      @master_socket
+      @state.master_socket
     end
 
     # Public: The pid of the running listener process.
-    attr_reader :pid
+    def pid
+      @state.pid
+    end
 
     # Public: Start the listener process.
     def run
       return if pid
+
       listener_socket, master_socket = UNIXSocket.pair
-      if @pid = fork
+      if @state.pid = fork
         # master
         listener_socket.close
         master_socket.close_on_exec = true
-        log "Started listener #{@pid}"
-        @master_socket = master_socket
+        log "Started listener #{@state.pid}"
+        @state.master_socket = master_socket
       else
         # listener
         master_socket.close
-        Master::TRAPS.each { |signal| trap(signal, 'DEFAULT') rescue nil }
-        Listener.new(@options.merge(:socket => listener_socket)).exec
+        Master::TRAPS.each { |signal| trap(signal, "DEFAULT") rescue nil }
+        Listener.new(@state.options.merge(socket: listener_socket)).exec
         exit
       end
     end
@@ -57,46 +62,50 @@ module Resqued
 
     # Public: Get the list of workers running from this listener.
     def running_workers
-      worker_pids.map { |pid, queue| { :pid => pid, :queue => queue } }
+      worker_pids.map { |pid, queue_key| { pid: pid, queue_key: queue_key } }
     end
 
     # Private: Map worker pids to queue names
     def worker_pids
-      @worker_pids ||= {}
+      @state.worker_pids ||= {}
     end
 
     # Public: Check for updates on running worker information.
     def read_worker_status(options)
       on_activity = options[:on_activity]
-      until @master_socket.nil?
-        IO.select([@master_socket], nil, nil, 0) or return
-        case line = @master_socket.readline
+      until @state.master_socket.nil?
+        IO.select([@state.master_socket], nil, nil, 0) or return
+        case line = @state.master_socket.readline
         when /^\+(\d+),(.*)$/
           worker_pids[$1] = $2
-          on_activity.worker_started($1) if on_activity
+          on_activity&.worker_started($1)
         when /^-(\d+)$/
           worker_pids.delete($1)
-          on_activity.worker_finished($1) if on_activity
+          on_activity&.worker_finished($1)
         when /^RUNNING/
-          on_activity.listener_running(self) if on_activity
-        when ''
+          on_activity&.listener_running(self)
+        when ""
           break
         else
           log "Malformed data from listener: #{line.inspect}"
         end
       end
     rescue EOFError, Errno::ECONNRESET
-      @master_socket.close
-      @master_socket = nil
+      @state.master_socket.close
+      @state.master_socket = nil
     end
 
     # Public: Tell the listener process that a worker finished.
     def worker_finished(pid)
-      return if @master_socket.nil?
-      @master_socket.puts(pid)
+      return if @state.master_socket.nil?
+
+      @state.master_socket.write_nonblock("#{pid}\n")
+    rescue IO::WaitWritable
+      log "Couldn't tell #{@state.pid} that #{pid} exited!"
+      # Ignore it, maybe the next time it'll work.
     rescue Errno::EPIPE
-      @master_socket.close
-      @master_socket = nil
+      @state.master_socket.close
+      @state.master_socket = nil
     end
   end
 end

data/lib/resqued/listener_state.rb

@@ -0,0 +1,8 @@
+module Resqued
+  class ListenerState
+    attr_accessor :master_socket
+    attr_accessor :options
+    attr_accessor :pid
+    attr_accessor :worker_pids
+  end
+end

data/lib/resqued/logging.rb

@@ -1,4 +1,4 @@
-require 'mono_logger'
+require "mono_logger"
 
 module Resqued
   # Mixin for any class that wants to write messages to the log file.
@@ -18,27 +18,34 @@ module Resqued
 
       class ResquedLogFormatter < ::Logger::Formatter
         def call(severity, time, progname, msg)
-          "[%s#%6d] %5s %s -- %s\n" % [format_datetime(time), $$, severity, progname, msg2str(msg)]
+          sprintf "[%s#%6d] %5s %s -- %s\n",
+                  format_datetime(time),
+                  $$,
+                  severity,
+                  progname,
+                  msg2str(msg)
         end
       end
 
       # Private: Lets our logger reopen its logfile without monologger EVEN KNOWING.
       class ResquedLoggingIOWrapper
+        # rubocop: disable Style/MethodMissingSuper
         def method_missing(*args)
           ::Resqued::Logging.logging_io.send(*args)
         end
+        # rubocop: enable Style/MethodMissingSuper
 
-        def respond_to?(*args)
-          ::Resqued::Logging.logging_io.respond_to?(*args)
+        def respond_to_missing?(method, *)
+          ::Resqued::Logging.logging_io.respond_to?(method)
         end
       end
 
       # Private: Get an IO to write log messages to.
       def logging_io
-        @logging_io = nil if @logging_io && @logging_io.closed?
+        @logging_io = nil if @logging_io&.closed?
         @logging_io ||=
           if path = Resqued::Logging.log_file
-            File.open(path, 'a').tap do |f|
+            File.open(path, "a").tap do |f|
               f.sync = true
               f.close_on_exec = true
               # Make sure we're not holding onto a stale filehandle.
@@ -60,13 +67,13 @@ module Resqued
 
       # Public.
       def log_file=(path)
-        ENV['RESQUED_LOGFILE'] = File.expand_path(path)
+        ENV["RESQUED_LOGFILE"] = File.expand_path(path)
         close_log
       end
 
       # Public.
       def log_file
-        ENV['RESQUED_LOGFILE']
+        ENV["RESQUED_LOGFILE"]
       end
     end
 

data/lib/resqued/master.rb

@@ -1,9 +1,11 @@
-require 'resqued/backoff'
-require 'resqued/listener_proxy'
-require 'resqued/logging'
-require 'resqued/pidfile'
-require 'resqued/procline_version'
-require 'resqued/sleepy'
+require "resqued/backoff"
+require "resqued/exec_on_hup"
+require "resqued/listener_pool"
+require "resqued/logging"
+require "resqued/master_state"
+require "resqued/pidfile"
+require "resqued/procline_version"
+require "resqued/sleepy"
 
 module Resqued
   # The master process.
@@ -16,19 +18,17 @@ module Resqued
     include Resqued::ProclineVersion
     include Resqued::Sleepy
 
-    def initialize(options)
-      @config_paths = options.fetch(:config_paths)
-      @pidfile      = options.fetch(:master_pidfile) { nil }
-      @status_pipe  = options.fetch(:status_pipe) { nil }
-      @fast_exit    = options.fetch(:fast_exit) { false }
+    def initialize(state, options = {})
+      @state = state
+      @status_pipe = options.fetch(:status_pipe, nil)
+      @listeners = ListenerPool.new(state)
       @listener_backoff = Backoff.new
-      @listeners_created = 0
     end
 
     # Public: Starts the master process.
     def run(ready_pipe = nil)
       report_unexpected_exits
-      with_pidfile(@pidfile) do
+      with_pidfile(@state.pidfile) do
         write_procline
         install_signal_handlers
         if ready_pipe
@@ -42,32 +42,39 @@ module Resqued
 
     # Private: dat main loop.
     def go_ham
+      # If we're resuming, we'll want to recycle the existing listener now.
+      prepare_new_listener
+
       loop do
         read_listeners
         reap_all_listeners(Process::WNOHANG)
-        start_listener unless @paused
+        start_listener unless @state.paused
         case signal = SIGNAL_QUEUE.shift
         when nil
           yawn(@listener_backoff.how_long? || 30.0)
         when :INFO
           dump_object_counts
         when :HUP
+          if @state.exec_on_hup
+            log "Execing a new master"
+            ExecOnHUP.exec!(@state)
+          end
           reopen_logs
           log "Restarting listener with new configuration and application."
           prepare_new_listener
         when :USR2
           log "Pause job processing"
-          @paused = true
-          kill_listener(:QUIT, @current_listener)
-          @current_listener = nil
+          @state.paused = true
+          kill_listener(:QUIT, @listeners.current)
+          @listeners.clear_current!
         when :CONT
           log "Resume job processing"
-          @paused = false
+          @state.paused = false
           kill_all_listeners(:CONT)
         when :INT, :TERM, :QUIT
           log "Shutting down..."
           kill_all_listeners(signal)
-          wait_for_workers unless @fast_exit
+          wait_for_workers unless @state.fast_exit
           break
         end
       end
@@ -85,14 +92,14 @@ module Resqued
       end
       top = 10
       log "#{total} objects. top #{top}:"
-      counts.sort_by { |name, count| count }.reverse.each_with_index do |(name, count), i|
-        if i < top
-          diff = ""
-          if last = @last_counts && @last_counts[name]
-            diff = " (#{'%+d' % (count - last)})"
-          end
-          log "   #{count} #{name}#{diff}"
+      counts.sort_by { |_, count| -count }.each_with_index do |(name, count), i|
+        next unless i < top
+
+        diff = ""
+        if last = @last_counts && @last_counts[name]
+          diff = sprintf(" (%+d)", (count - last))
         end
+        log "   #{count} #{name}#{diff}"
       end
       @last_counts = counts
       log GC.stat.inspect
@@ -100,59 +107,45 @@ module Resqued
       log "Error while counting objects: #{e}"
     end
 
-    # Private: Map listener pids to ListenerProxy objects.
-    def listener_pids
-      @listener_pids ||= {}
-    end
-
-    # Private: All the ListenerProxy objects.
-    def all_listeners
-      listener_pids.values
-    end
-
     def start_listener
-      return if @current_listener || @listener_backoff.wait?
-      @current_listener = ListenerProxy.new(:config_paths => @config_paths, :old_workers => all_listeners.map { |l| l.running_workers }.flatten, :listener_id => next_listener_id)
-      @current_listener.run
-      listener_status @current_listener, 'start'
+      return if @listeners.current || @listener_backoff.wait?
+
+      listener = @listeners.start!
+      listener_status listener, "start"
       @listener_backoff.started
-      listener_pids[@current_listener.pid] = @current_listener
       write_procline
     end
 
-    def next_listener_id
-      @listeners_created += 1
-    end
-
     def read_listeners
-      all_listeners.each do |l|
-        l.read_worker_status(:on_activity => self)
+      @listeners.each do |l|
+        l.read_worker_status(on_activity: self)
       end
     end
 
     # Listener message: A worker just started working.
     def worker_started(pid)
-      worker_status(pid, 'start')
+      worker_status(pid, "start")
     end
 
     # Listener message: A worker just stopped working.
     #
     # Forwards the message to the other listeners.
     def worker_finished(pid)
-      worker_status(pid, 'stop')
-      all_listeners.each do |other|
+      worker_status(pid, "stop")
+      @listeners.each do |other|
         other.worker_finished(pid)
       end
     end
 
-    # Listener message: A listener finished booting, and is ready to start workers.
+    # Listener message: A listener finished booting, and is ready to
+    # start workers.
     #
     # Promotes a booting listener to be the current listener.
     def listener_running(listener)
-      listener_status(listener, 'ready')
-      if listener == @current_listener
-        kill_listener(:QUIT, @last_good_listener)
-        @last_good_listener = nil
+      listener_status(listener, "ready")
+      if listener == @listeners.current
+        kill_listener(:QUIT, @listeners.last_good)
+        @listeners.clear_last_good!
       else
         # This listener didn't receive the last SIGQUIT we sent.
         # (It was probably sent before the listener had set up its traps.)
@@ -165,23 +158,26 @@ module Resqued
     #
     # The old one will be killed when the new one is ready for workers.
     def prepare_new_listener
-      if @last_good_listener
-        # The last_good_listener is still running because we got another HUP before the new listener finished booting.
-        # Keep the last_good_listener (where all the workers are) and kill the booting current_listener. We'll start a new one.
-        kill_listener(:QUIT, @current_listener)
+      if @listeners.last_good
+        # The last good listener is still running because we got another
+        # HUP before the new listener finished booting.
+        # Keep the last_good_listener (where all the workers are) and
+        # kill the booting current_listener. We'll start a new one.
+        kill_listener(:QUIT, @listeners.current)
+        # Indicate to `start_listener` that it should start a new
+        # listener.
+        @listeners.clear_current!
       else
-        @last_good_listener = @current_listener
+        @listeners.cycle_current
       end
-      # Indicate to `start_listener` that it should start a new listener.
-      @current_listener = nil
     end
 
     def kill_listener(signal, listener)
-      listener.kill(signal) if listener
+      listener&.kill(signal)
     end
 
     def kill_all_listeners(signal)
-      all_listeners.each do |l|
+      @listeners.each do |l|
         l.kill(signal)
       end
     end
@@ -191,80 +187,80 @@ module Resqued
     end
 
     def reap_all_listeners(waitpid_flags = 0)
-      begin
-        lpid, status = Process.waitpid2(-1, waitpid_flags)
-        if lpid
+      until @listeners.empty?
+        begin
+          lpid, status = Process.waitpid2(-1, waitpid_flags)
+          return unless lpid
+
           log "Listener exited #{status}"
-          if @current_listener && @current_listener.pid == lpid
+
+          if @listeners.current_pid == lpid
             @listener_backoff.died
-            @current_listener = nil
+            @listeners.clear_current!
+          end
+
+          if @listeners.last_good_pid == lpid
+            @state.clear_last_good!
           end
 
-          if @last_good_listener && @last_good_listener.pid == lpid
-            @last_good_listener = nil
+          if dead_listener = @listeners.delete(lpid)
+            listener_status dead_listener, "stop"
+            dead_listener.dispose
           end
-          dead_listener = listener_pids.delete(lpid)
-          listener_status dead_listener, 'stop'
-          dead_listener.dispose
+
           write_procline
-        else
+        rescue Errno::ECHILD
           return
         end
-      rescue Errno::ECHILD
-        return
-      end while true
+      end
     end
 
-    SIGNALS = [ :HUP, :INT, :USR2, :CONT, :TERM, :QUIT ]
-    OPTIONAL_SIGNALS = [ :INFO ]
-    OTHER_SIGNALS = [:CHLD, 'EXIT']
+    SIGNALS = [:HUP, :INT, :USR2, :CONT, :TERM, :QUIT].freeze
+    OPTIONAL_SIGNALS = [:INFO].freeze
+    OTHER_SIGNALS = [:CHLD, "EXIT"].freeze
     TRAPS = SIGNALS + OPTIONAL_SIGNALS + OTHER_SIGNALS
 
-    SIGNAL_QUEUE = []
+    SIGNAL_QUEUE = [] # rubocop: disable Style/MutableConstant
 
     def install_signal_handlers
       trap(:CHLD) { awake }
-      SIGNALS.each { |signal| trap(signal) { SIGNAL_QUEUE << signal ; awake } }
-      OPTIONAL_SIGNALS.each { |signal| trap(signal) { SIGNAL_QUEUE << signal ; awake } rescue nil }
+      SIGNALS.each { |signal| trap(signal) { SIGNAL_QUEUE << signal; awake } }
+      OPTIONAL_SIGNALS.each { |signal| trap(signal) { SIGNAL_QUEUE << signal; awake } rescue nil }
     end
 
     def report_unexpected_exits
-      trap('EXIT') do
+      trap("EXIT") do
         log("EXIT #{$!.inspect}")
-        if $!
-          $!.backtrace.each do |line|
-            log(line)
-          end
+        $!&.backtrace&.each do |line|
+          log(line)
         end
       end
     end
 
     def no_more_unexpected_exits
-      trap('EXIT', 'DEFAULT')
+      trap("EXIT", "DEFAULT")
     end
 
     def yawn(duration)
-      super(duration, all_listeners.map { |l| l.read_pipe })
+      super(duration, @listeners.map { |l| l.read_pipe })
     end
 
     def write_procline
-      $0 = "#{procline_version} master [gen #{@listeners_created}] [#{listener_pids.size} running] #{ARGV.join(' ')}"
+      $0 = "#{procline_version} master [gen #{@state.listeners_created}] [#{@listeners.size} running] #{ARGV.join(' ')}"
     end
 
     def listener_status(listener, status)
-      if listener && listener.pid
-        status_message('listener', listener.pid, status)
+      if listener&.pid
+        status_message("listener", listener.pid, status)
       end
     end
 
     def worker_status(pid, status)
-      status_message('worker', pid, status)
+      status_message("worker", pid, status)
     end
 
     def status_message(type, pid, status)
-      if @status_pipe
-        @status_pipe.write("#{type},#{pid},#{status}\n")
-      end
+      @status_pipe&.write("#{type},#{pid},#{status}\n")
     end
   end
 end