resqued 0.9.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: acc537c28c318ee1628ea4c14911a4716c0d8330
-  data.tar.gz: 7b625fcb530e90ca12459f75e7529641dd660758
+SHA256:
+  metadata.gz: 606d7e3fb51d8595ce4a8e4fe7bdb28fc3aa73323652cbb98f9e0db19e982960
+  data.tar.gz: 87d0126e0b4f6e804a63d680d6946aa83621d19ddfa22c9427772d84eb9208df
 SHA512:
-  metadata.gz: 68d8f563141474b5b4e8be2c300763cd34052bfd2da431f2455f01cdbc36abfd9c0cf0d73b87bc599620a6adc0461d246717845a64d4cfe6707d49d2c725cb5b
-  data.tar.gz: 23faeaefbffacca556681d465bb9de03d07ff73bd2ab135788e8263998e483db49dbc4d6b75b5b6bfc1d5922634a2427590a34e2134e3d5c28470585ff514d9f
+  metadata.gz: abcd739a5f7d36a1e980b055e90e36cd63d0303264126f35a27ef260717022fda2ddd23630b13a954d93cb2f038067ae9d2579347b5ca1a48a19ab18528d5a2e
+  data.tar.gz: 82b66d4c943da3c73e5ae0459197fad97931770eeb3d6acd77b556f69d285e2c8666823b2f9fd563d25337974012d81b5e8d4f2f64b5f39d835e7c8dbada409c

data/CHANGES.md

@@ -1,5 +1,28 @@
 Starting with version 0.6.1, resqued uses semantic versioning to indicate incompatibilities between the master process, listener process, and configuration.
 
+v0.11.0
+-------
+* Ignore SIGHUP in Listener and Worker processes. (#61)
+
+v0.10.3
+-------
+* Fix a timing related crash during reload. (#60)
+
+v0.10.2
+-------
+* Shut down cleanly even if there are other stray child processes of the master. (#59)
+
+v0.10.1
+-------
+* Avoid deadlock if a listener stops responding. (#58)
+* When using 'percent' for a queue in a worker pool, always assign at least one worker. (#57)
+
+v0.10.0
+-------
+* Master process restarts itself (#51), so that it doesn't continue running stale code indefinitely. This will help with the rollout process when changes like #50 are introduced, so that the master process will catch up. The risk of this is that the master process might not be able to be restarted, which would lead to it crashing. The mostly likely way for that to happen is if you try to roll back your version of resqued to 0.9.0 or earlier. If you need to do that, ensure that your process monitor (systemd, god, etc.) is able to restart the master process. You can disable the new behavior by passing `--no-exec-on-hup`.
+* Added rubocop. (#52)
+* Changed supported (read: tested in CI) ruby versions from [2.0 .. 2.3] to [2.3 .. 2.6].
+
 v0.9.0
 ------
 * Avoid Errno::E2BIG on SIGHUP when there are lots of workers and lots of queues per worker. (#50) This changes the format of an env var that master passes to listener. Old and new versions won't crash, but they won't be able to communicate about currenly running workers.

data/docs/signals.md

@@ -34,6 +34,8 @@ The Listener process forwards `SIGCONT` to all of its workers.
 
 The Listener process handles `SIGINT`, `SIGTERM`, and `SIGQUIT`. When it receives one of these signals, it goes into shutdown mode. It sends the received signal to all of its workers. When all workers have exited, the Listener process exits.
 
+The Listener process handles `SIGHUP` and does nothing. This makes it easier to reload resqued in a docker container, since many container platforms will send a requested signal to all processes in the container.
+
 ## Worker
 
 The Worker process uses resque's signal handling. Resque 1.23.0 handles the following signals:
@@ -44,3 +46,5 @@ The Worker process uses resque's signal handling. Resque 1.23.0 handles the foll
 * `USR1`: Kill the forked child immediately, continue processing jobs.
 * `USR2`: Don't process any new jobs
 * `CONT`: Start processing jobs again after a USR2
+
+Resqued leaves a handler for `HUP` in place that does nothing. This makes it easier to reload resqued in a docker container, since many container platforms will send a requested signal to all processes in the container.

data/exe/resqued

@@ -1,74 +1,93 @@
 #!/usr/bin/env ruby
 
-if ARGV[0] == 'listener'
-  require 'resqued/listener'
+if ARGV[0] == "listener"
+  require "resqued/listener"
   Resqued::Listener.exec!
   exit 0
 end
 
-require 'optparse'
+require "optparse"
 
-options = {}
+options = { exec_on_hup: true }
 daemonize = false
 test = false
 
-opts = OptionParser.new do |opts|
+opts = OptionParser.new do |opts| # rubocop: disable Lint/ShadowingOuterLocalVariable
   opts.banner = "Usage: resqued [options] resqued-config-files..."
 
-  opts.on '-h', '--help', 'Show this message' do
+  opts.on "-h", "--help", "Show this message" do
     puts opts
     exit
   end
 
-  opts.on '-v', '--version', 'Show the version' do
-    require 'resqued/version'
+  opts.on "-v", "--version", "Show the version" do
+    require "resqued/version"
     puts Resqued::VERSION
     exit
   end
 
-  opts.on '--test', 'Report which worker would start' do
+  opts.on "--test", "Report which worker would start" do
     test = true
   end
 
-  opts.on '--fast-exit', 'Exit quickly on SIGQUIT, SIGTERM' do
+  opts.on "--fast-exit", "Exit quickly on SIGQUIT, SIGTERM" do
     options[:fast_exit] = true
   end
 
-  opts.on '-p', '--pidfile PIDFILE', 'Store the pid of the master process in PIDFILE' do |v|
+  opts.on "-p", "--pidfile PIDFILE", "Store the pid of the master process in PIDFILE" do |v|
     options[:master_pidfile] = v
   end
 
-  opts.on '-l', '--logfile LOGFILE', 'Write output to LOGFILE instead of stdout' do |v|
-    require 'resqued/logging'
+  opts.on "-l", "--logfile LOGFILE", "Write output to LOGFILE instead of stdout" do |v|
+    require "resqued/logging"
     Resqued::Logging.log_file = v
   end
 
-  opts.on '-D', '--daemonize', 'Run daemonized in the background' do
+  opts.on "-D", "--daemonize", "Run daemonized in the background" do
     daemonize = true
   end
+
+  opts.on "--no-exec-on-hup", "Do not re-exec the master process on SIGHUP" do
+    options[:exec_on_hup] = false
+  end
+
+  opts.on "--replace FILE", "(internal)" do |v|
+    options[:master_state] = v
+  end
 end
 
 opts.parse!
 options[:config_paths] = ARGV
 
-if options[:config_paths].size == 0
-  puts opts
-  exit 1
+def require_config_paths!(options, opts)
+  if options[:config_paths].empty?
+    puts opts
+    exit 1
+  end
 end
 
 if test
-  require 'resqued/config'
+  require_config_paths! options, opts
+  require "resqued/config"
   workers = Resqued::Config.new(options[:config_paths]).build_workers
   puts "Workers defined in #{options[:config_paths].join(' ')}"
   workers.each_with_index do |worker, index|
     puts "#{index + 1}: #{worker.queues.join(',')}"
   end
 else
-  require 'resqued'
-  Resqued.capture_start_ctx!
-  resqued = Resqued::Master.new(options)
+  require "resqued"
+  state = Resqued::MasterState.new
+  if options[:master_state]
+    Resqued::Logging.logger.info "Resuming master from #{options[:master_state]}"
+    Resqued::ExecOnHUP.restore_state(state, options[:master_state])
+  else
+    require_config_paths! options, opts
+    Resqued.capture_start_ctx!
+    state.init(options)
+  end
+  resqued = Resqued::Master.new(state)
   if daemonize
-    require 'resqued/daemon'
+    require "resqued/daemon"
     resqued = Resqued::Daemon.new(resqued)
   end
   resqued.run

data/lib/resqued.rb

@@ -1,12 +1,12 @@
-require 'resqued/master'
-require 'resqued/version'
+require "resqued/master"
+require "resqued/version"
 
 module Resqued
-  START_CTX = {}
+  START_CTX = {} # rubocop: disable Style/MutableConstant
 
   def self.capture_start_ctx!
-    START_CTX['$0'] = $0.dup
-    START_CTX['pwd'] =
+    START_CTX["$0"] = $0.dup
+    START_CTX["pwd"] =
       begin
         env_pwd = ENV["PWD"]
         env_pwd_stat = File.stat env_pwd

data/lib/resqued/config.rb

@@ -1,6 +1,6 @@
-require 'resqued/config/after_fork'
-require 'resqued/config/before_fork'
-require 'resqued/config/worker'
+require "resqued/config/after_fork"
+require "resqued/config/before_fork"
+require "resqued/config/worker"
 
 module Resqued
   module Config
@@ -14,22 +14,22 @@ module Resqued
     # Does the things that the config file says to do.
     class Configuration
       def initialize(config_paths)
-        @config_data = config_paths.map { |path| {:content => File.read(path), :path => path} }
+        @config_data = config_paths.map { |path| { content: File.read(path), path: path } }
       end
 
       # Public: Performs the `before_fork` action from the config.
       def before_fork(resqued)
-        Resqued::Config::BeforeFork.new(:resqued => resqued).apply_all(@config_data)
+        Resqued::Config::BeforeFork.new(resqued: resqued).apply_all(@config_data)
       end
 
       # Public: Performs the `after_fork` action from the config.
       def after_fork(worker)
-        Resqued::Config::AfterFork.new(:worker => worker).apply_all(@config_data)
+        Resqued::Config::AfterFork.new(worker: worker).apply_all(@config_data)
       end
 
       # Public: Builds the workers specified in the config.
       def build_workers
-        Resqued::Config::Worker.new(:config => self).apply_all(@config_data)
+        Resqued::Config::Worker.new(config: self).apply_all(@config_data)
       end
     end
   end

data/lib/resqued/config/after_fork.rb

@@ -1,4 +1,4 @@
-require 'resqued/config/base'
+require "resqued/config/base"
 
 module Resqued
   module Config

data/lib/resqued/config/base.rb

@@ -1,4 +1,4 @@
-require 'resqued/config/dsl'
+require "resqued/config/dsl"
 
 module Resqued
   module Config

data/lib/resqued/config/before_fork.rb

@@ -1,4 +1,4 @@
-require 'resqued/config/base'
+require "resqued/config/base"
 
 module Resqued
   module Config

data/lib/resqued/config/worker.rb

@@ -1,5 +1,5 @@
-require 'resqued/config/base'
-require 'resqued/worker'
+require "resqued/config/base"
+require "resqued/worker"
 
 module Resqued
   module Config
@@ -21,9 +21,9 @@ module Resqued
       def worker(*queues)
         options = queues.last.is_a?(Hash) ? queues.pop.dup : {}
         queues = queues.flatten
-        queues = ['*'] if queues.empty?
+        queues = ["*"] if queues.empty?
         queues = queues.shuffle if options.delete(:shuffle_queues)
-        @workers << @worker_class.new(options.merge(@worker_options).merge(:queues => queues))
+        @workers << @worker_class.new(options.merge(@worker_options).merge(queues: queues))
       end
 
       # DSL: Set up a pool of workers. Define queues for the members of the pool with `queue`.
@@ -81,15 +81,16 @@ module Resqued
       # on the concurrency values established and the total number of workers.
       def build_pool_workers!
         return unless @pool_size
+
         queues = _fixed_concurrency_queues
         1.upto(@pool_size) do |worker_num|
-          queue_names = queues.
-            select { |name, concurrency| concurrency >= worker_num }.
-            map { |name, _| name }
+          queue_names = queues
+                        .select { |_name, concurrency| concurrency >= worker_num }
+                        .map { |name, _concurrency| name }
           if queue_names.any?
             worker(queue_names, @pool_options)
           else
-            worker('*', @pool_options)
+            worker("*", @pool_options)
           end
         end
       end
@@ -106,13 +107,12 @@ module Resqued
       # values (between 0.0 and 1.0). The value may also be nil, in which case the
       # maximum worker_processes value is returned.
       def _translate_concurrency_value(value)
-        case
-        when value.nil?
+        if value.nil?
           @pool_size
-        when value.is_a?(1.class)
+        elsif value.is_a?(1.class)
           value < @pool_size ? value : @pool_size
-        when value.is_a?(Float) && value >= 0.0 && value <= 1.0
-          (@pool_size * value).to_i
+        elsif value.is_a?(Float) && value >= 0.0 && value <= 1.0
+          [(@pool_size * value).to_i, 1].max
         else
           raise TypeError, "Unknown concurrency value: #{value.inspect}"
         end

data/lib/resqued/daemon.rb

@@ -12,6 +12,7 @@ module Resqued
         wr.close
         begin
           master_pid = rd.readpartial(16).to_i
+          puts "Started master: #{master_pid}" if ENV["DEBUG"]
           exit
         rescue EOFError
           puts "Master process failed to start!"

data/lib/resqued/exec_on_hup.rb

@@ -0,0 +1,43 @@
+require "tempfile"
+require "yaml"
+
+module Resqued
+  class ExecOnHUP
+    # Public: Replace the current master process with a new one, while preserving state.
+    def self.exec!(state)
+      exec Resqued::START_CTX["$0"], "--replace", store_state(state), exec_opts(state)
+    end
+
+    # Internal: Returns exec options for each open socket in 'state'.
+    def self.exec_opts(state)
+      exec_opts = {}
+      state.sockets.each do |sock|
+        exec_opts[sock.to_i] = sock
+      end
+      if pwd = Resqued::START_CTX["pwd"]
+        exec_opts[:chdir] = pwd
+      end
+      return exec_opts
+    end
+
+    # Internal: Write out current state to a file, so that a new master can pick up from where we left off.
+    def self.store_state(state)
+      data = { version: Resqued::VERSION }
+      data[:start_ctx] = Resqued::START_CTX
+      data[:state] = state.to_h
+
+      f = Tempfile.create "resqued-state"
+      f.write(YAML.dump(data))
+      f.close
+      return f.path
+    end
+
+    # Internal: Restore the master's state, and remove the state file.
+    def self.restore_state(state, path)
+      data = YAML.safe_load(File.read(path), [Symbol], [], true)
+      Resqued::START_CTX.replace(data[:start_ctx] || {})
+      state.restore(data[:state])
+      File.unlink(path) rescue nil
+    end
+  end
+end

data/lib/resqued/listener.rb

@@ -1,12 +1,12 @@
-require 'socket'
+require "socket"
 
-require 'resqued/config'
-require 'resqued/logging'
-require 'resqued/procline_version'
-require 'resqued/runtime_info'
-require 'resqued/sleepy'
-require 'resqued/version'
-require 'resqued/worker'
+require "resqued/config"
+require "resqued/logging"
+require "resqued/procline_version"
+require "resqued/runtime_info"
+require "resqued/sleepy"
+require "resqued/version"
+require "resqued/worker"
 
 module Resqued
   # A listener process. Watches resque queues and forks workers.
@@ -30,67 +30,70 @@ module Resqued
     # Runs in the master process.
     def exec
       socket_fd = @socket.to_i
-      ENV['RESQUED_SOCKET']      = socket_fd.to_s
-      ENV['RESQUED_CONFIG_PATH'] = @config_paths.join(':')
-      ENV['RESQUED_STATE']       = (@old_workers.map { |r| "#{r[:pid]}|#{r[:queue_key]}" }.join('||'))
-      ENV['RESQUED_LISTENER_ID'] = @listener_id.to_s
-      ENV['RESQUED_MASTER_VERSION'] = Resqued::VERSION
+      ENV["RESQUED_SOCKET"]      = socket_fd.to_s
+      ENV["RESQUED_CONFIG_PATH"] = @config_paths.join(":")
+      ENV["RESQUED_STATE"]       = @old_workers.map { |r| "#{r[:pid]}|#{r[:queue_key]}" }.join("||")
+      ENV["RESQUED_LISTENER_ID"] = @listener_id.to_s
+      ENV["RESQUED_MASTER_VERSION"] = Resqued::VERSION
       log "exec: #{Resqued::START_CTX['$0']} listener"
-      exec_opts = {socket_fd => socket_fd} # Ruby 2.0 needs to be told to keep the file descriptor open during exec.
-      if start_pwd = Resqued::START_CTX['pwd']
+      exec_opts = { socket_fd => socket_fd } # Ruby 2.0 needs to be told to keep the file descriptor open during exec.
+      if start_pwd = Resqued::START_CTX["pwd"]
         exec_opts[:chdir] = start_pwd
       end
-      procline_buf = ' ' * 256 # make room for setproctitle
-      Kernel.exec(Resqued::START_CTX['$0'], 'listener', procline_buf, exec_opts)
+      procline_buf = " " * 256 # make room for setproctitle
+      Kernel.exec(Resqued::START_CTX["$0"], "listener", procline_buf, exec_opts)
     end
 
     # Public: Given args from #exec, start this listener.
     def self.exec!
       options = {}
-      if socket = ENV['RESQUED_SOCKET']
+      if socket = ENV["RESQUED_SOCKET"]
         options[:socket] = Socket.for_fd(socket.to_i)
       end
-      if path = ENV['RESQUED_CONFIG_PATH']
-        options[:config_paths] = path.split(':')
+      if path = ENV["RESQUED_CONFIG_PATH"]
+        options[:config_paths] = path.split(":")
       end
-      if state = ENV['RESQUED_STATE']
-        options[:old_workers] = state.split('||').map { |s| Hash[[:pid,:queue_key].zip(s.split('|'))] }
+      if state = ENV["RESQUED_STATE"]
+        options[:old_workers] = state.split("||").map { |s| Hash[[:pid, :queue_key].zip(s.split("|"))] }
       end
-      if listener_id = ENV['RESQUED_LISTENER_ID']
+      if listener_id = ENV["RESQUED_LISTENER_ID"]
         options[:listener_id] = listener_id
       end
       new(options).run
     end
 
-    SIGNALS = [ :CONT, :QUIT, :INT, :TERM ]
-    ALL_SIGNALS = SIGNALS + [ :CHLD ]
+    SIGNALS = [:CONT, :QUIT, :INT, :TERM].freeze
+    ALL_SIGNALS = SIGNALS + [:CHLD, :HUP]
 
-    SIGNAL_QUEUE = []
+    SIGNAL_QUEUE = [] # rubocop: disable Style/MutableConstant
 
     # Public: Run the main loop.
     def run
+      trap(:HUP) {} # ignore this, in case it trickles in from the master.
       trap(:CHLD) { awake }
-      SIGNALS.each { |signal| trap(signal) { SIGNAL_QUEUE << signal ; awake } }
+      SIGNALS.each { |signal| trap(signal) { SIGNAL_QUEUE << signal; awake } }
       @socket.close_on_exec = true
-      write_procline('starting')
+      write_procline("starting")
 
       config = Resqued::Config.new(@config_paths)
       set_default_resque_logger
       config.before_fork(info)
       report_to_master("RUNNING")
 
-      write_procline('running')
+      write_procline("running")
       init_workers(config)
       exit_signal = run_workers_run
 
-      write_procline('shutdown')
+      write_procline("shutdown")
       burn_down_workers(exit_signal || :QUIT)
+      @socket.close
+      @socket = nil
     end
 
     # Private.
     def set_default_resque_logger
-      require 'resque'
-      if Resque.respond_to?('logger=')
+      require "resque"
+      if Resque.respond_to?("logger=")
         Resque.logger = Resqued::Logging.build_logger
       end
     end
@@ -101,7 +104,7 @@ module Resqued
         reap_workers(Process::WNOHANG)
         check_for_expired_workers
         start_idle_workers
-        write_procline('running')
+        write_procline("running")
         case signal = SIGNAL_QUEUE.shift
         when nil
           yawn
@@ -119,10 +122,11 @@ module Resqued
     def burn_down_workers(signal)
       loop do
         check_for_expired_workers
-        write_procline('shutdown')
+        write_procline("shutdown")
         SIGNAL_QUEUE.clear
 
         break if :no_child == reap_workers(Process::WNOHANG)
+
         kill_all(signal)
 
         sleep 1 # Don't kill any more often than every 1s.
@@ -134,7 +138,7 @@ module Resqued
 
     # Private: send a signal to all the workers.
     def kill_all(signal)
-      idle, running = partition_workers
+      running = running_workers
       log "kill -#{signal} #{running.map { |r| r.pid }.inspect}"
       running.each { |worker| worker.kill(signal) }
     end
@@ -171,13 +175,11 @@ module Resqued
     def reap_workers(waitpidflags = 0)
       loop do
         worker_pid, status = Process.waitpid2(-1, waitpidflags)
-        if worker_pid.nil?
-          return :none_ready
-        else
-          log "Worker exited #{status}"
-          finish_worker(worker_pid, status)
-          report_to_master("-#{worker_pid}")
-        end
+        return :none_ready if worker_pid.nil?
+
+        log "Worker exited #{status}"
+        finish_worker(worker_pid, status)
+        report_to_master("-#{worker_pid}")
       end
     rescue Errno::ECHILD
       # All done
@@ -187,6 +189,7 @@ module Resqued
     # Private: Check if master reports any dead workers.
     def check_for_expired_workers
       return unless @socket
+
       loop do
         IO.select([@socket], nil, nil, 0) or return
         line = @socket.readline
@@ -210,11 +213,11 @@ module Resqued
     # Private.
     def start_idle_workers
       workers.each do |worker|
-        if worker.idle?
-          worker.try_start
-          if pid = worker.pid
-            report_to_master("+#{pid},#{worker.queue_key}")
-          end
+        next unless worker.idle?
+
+        worker.try_start
+        if pid = worker.pid
+          report_to_master("+#{pid},#{worker.queue_key}")
         end
       end
     end
@@ -236,7 +239,7 @@ module Resqued
     #     report_to_master("+12345,queue")  # Worker process PID:12345 started, working on a job from "queue".
     #     report_to_master("-12345")        # Worker process PID:12345 exited.
     def report_to_master(status)
-      @socket.puts(status) if @socket
+      @socket&.puts(status)
     rescue Errno::EPIPE => e
       @socket = nil
       log "#{e.class.name} while writing to master"