einhorn 0.7.3 → 1.0.0

data/lib/einhorn/command.rb

@@ -1,28 +1,25 @@
-require 'pp'
-require 'set'
-require 'tmpdir'
+require "pp"
+require "set"
+require "tmpdir"
 
-require 'einhorn/command/interface'
+require "einhorn/command/interface"
+require "einhorn/prctl"
 
 module Einhorn
   module Command
     def self.reap
-      begin
-        while true
-          Einhorn.log_debug('Going to reap a child process')
-
-          pid = Process.wait(-1, Process::WNOHANG)
-          return unless pid
-          mourn(pid)
-          Einhorn::Event.break_loop
-        end
-      rescue Errno::ECHILD
-      end
+      loop do
+        Einhorn.log_debug("Going to reap a child process")
+        pid = Process.wait(-1, Process::WNOHANG)
+        return unless pid
+        cleanup(pid)
+        Einhorn::Event.break_loop
+      end
+    rescue Errno::ECHILD
     end
 
-    # Mourn the death of your child
-    def self.mourn(pid)
-      unless spec = Einhorn::State.children[pid]
+    def self.cleanup(pid)
+      unless (spec = Einhorn::State.children[pid])
         Einhorn.log_error("Could not find any config for exited child #{pid.inspect}! This probably indicates a bug in Einhorn.")
         return
       end
@@ -32,7 +29,7 @@ module Einhorn
       # Unacked worker
       if spec[:type] == :worker && !spec[:acked]
         Einhorn::State.consecutive_deaths_before_ack += 1
-        extra = ' before it was ACKed'
+        extra = " before it was ACKed"
       else
         extra = nil
       end
@@ -40,11 +37,23 @@ module Einhorn
       case type = spec[:type]
       when :worker
         Einhorn.log_info("===> Exited worker #{pid.inspect}#{extra}", :upgrade)
+      when :state_passer
+        Einhorn.log_debug("===> Exited state passing process #{pid.inspect}", :upgrade)
       else
         Einhorn.log_error("===> Exited process #{pid.inspect} has unrecgonized type #{type.inspect}: #{spec.inspect}", :upgrade)
       end
     end
 
+    def self.register_ping(pid, request_id)
+      unless (spec = Einhorn::State.children[pid])
+        Einhorn.log_error("Could not find state for PID #{pid.inspect}; ignoring ACK.")
+        return
+      end
+
+      spec[:pinged_at] = Time.now
+      spec[:pinged_request_id] = request_id
+    end
+
     def self.register_manual_ack(pid)
       ack_mode = Einhorn::State.ack_mode
       unless ack_mode[:type] == :manual
@@ -73,7 +82,7 @@ module Einhorn
     end
 
     def self.register_ack(pid)
-      unless spec = Einhorn::State.children[pid]
+      unless (spec = Einhorn::State.children[pid])
         Einhorn.log_error("Could not find state for PID #{pid.inspect}; ignoring ACK.")
         return
       end
@@ -83,10 +92,8 @@ module Einhorn
         return
       end
 
-      if Einhorn::State.consecutive_deaths_before_ack > 0
-        extra = ", breaking the streak of #{Einhorn::State.consecutive_deaths_before_ack} consecutive unacked workers dying"
-      else
-        extra = nil
+      extra = if Einhorn::State.consecutive_deaths_before_ack > 0
+        ", breaking the streak of #{Einhorn::State.consecutive_deaths_before_ack} consecutive unacked workers dying"
       end
       Einhorn::State.consecutive_deaths_before_ack = 0
 
@@ -96,14 +103,14 @@ module Einhorn
       Einhorn::Event.break_loop
     end
 
-    def self.signal_all(signal, children=nil, record=true)
+    def self.signal_all(signal, children = nil, record = true)
       children ||= Einhorn::WorkerPool.workers
-
       signaled = {}
+
       Einhorn.log_info("Sending #{signal} to #{children.inspect}", :upgrade)
 
       children.each do |child|
-        unless spec = Einhorn::State.children[child]
+        unless (spec = Einhorn::State.children[child])
           Einhorn.log_error("Trying to send #{signal} to dead child #{child.inspect}. The fact we tried this probably indicates a bug in Einhorn.", :upgrade)
           next
         end
@@ -113,11 +120,13 @@ module Einhorn
             Einhorn.log_error("Re-sending #{signal} to already-signaled child #{child.inspect}. It may be slow to spin down, or it may be swallowing #{signal}s.", :upgrade)
           end
           spec[:signaled].add(signal)
+          spec[:last_signaled_at] = Time.now
         end
 
         begin
           Process.kill(signal, child)
         rescue Errno::ESRCH
+          Einhorn.log_debug("Attempted to #{signal} child #{child.inspect} but the process does not exist", :upgrade)
         else
           signaled[child] = spec
         end
@@ -127,7 +136,7 @@ module Einhorn
         Einhorn::Event::Timer.open(Einhorn::State.signal_timeout) do
           children.each do |child|
             spec = Einhorn::State.children[child]
-            next unless spec # Process is already dead and removed by mourn
+            next unless spec # Process is already dead and removed by cleanup
             signaled_spec = signaled[child]
             next unless signaled_spec # We got ESRCH when trying to signal
             if spec[:spinup_time] != signaled_spec[:spinup_time]
@@ -137,15 +146,15 @@ module Einhorn
 
             Einhorn.log_info("Child #{child.inspect} is still active after #{Einhorn::State.signal_timeout}s. Sending SIGKILL.")
             begin
-              Process.kill('KILL', child)
+              Process.kill("KILL", child)
             rescue Errno::ESRCH
             end
-            spec[:signaled].add('KILL')
+            spec[:signaled].add("KILL")
           end
         end
-      end
 
-      "Successfully sent #{signal}s to #{signaled.length} processes: #{signaled.keys}"
+        Einhorn.log_info("Successfully sent #{signal}s to #{signaled.length} processes: #{signaled.keys}")
+      end
     end
 
     def self.increment
@@ -153,14 +162,14 @@ module Einhorn
       old = Einhorn::State.config[:number]
       new = (Einhorn::State.config[:number] += 1)
       output = "Incrementing number of workers from #{old} -> #{new}"
-      $stderr.puts(output)
+      warn(output)
       output
     end
 
     def self.decrement
       if Einhorn::State.config[:number] <= 1
         output = "Can't decrease number of workers (already at #{Einhorn::State.config[:number]}).  Run kill #{$$} if you really want to kill einhorn."
-        $stderr.puts(output)
+        warn(output)
         return output
       end
 
@@ -168,7 +177,7 @@ module Einhorn
       old = Einhorn::State.config[:number]
       new = (Einhorn::State.config[:number] -= 1)
       output = "Decrementing number of workers from #{old} -> #{new}"
-      $stderr.puts(output)
+      warn(output)
       output
     end
 
@@ -181,12 +190,12 @@ module Einhorn
       old = Einhorn::State.config[:number]
       Einhorn::State.config[:number] = new
       output = "Altering worker count, #{old} -> #{new}. Will "
-      if old < new
-        output << "spin up additional workers."
+      output << if old < new
+        "spin up additional workers."
       else
-        output << "gracefully terminate workers."
+        "gracefully terminate workers."
       end
-      $stderr.puts(output)
+      warn(output)
       output
     end
 
@@ -197,8 +206,8 @@ module Einhorn
       end
 
       {
-        :state => global_state,
-        :persistent_descriptors => descriptor_state,
+        state: global_state,
+        persistent_descriptors: descriptor_state
       }
     end
 
@@ -218,6 +227,7 @@ module Einhorn
 
       fork do
         Einhorn::TransientState.whatami = :state_passer
+        Einhorn::State.children[Process.pid] = {type: :state_passer}
         Einhorn::State.generation += 1
         read.close
 
@@ -242,8 +252,8 @@ module Einhorn
 
       begin
         Einhorn.initialize_reload_environment
-        respawn_commandline = Einhorn.upgrade_commandline(['--with-state-fd', read.fileno.to_s])
-        respawn_commandline << { :close_others => false }
+        respawn_commandline = Einhorn.upgrade_commandline(["--with-state-fd", read.fileno.to_s])
+        respawn_commandline << {close_others: false}
         Einhorn.log_info("About to re-exec einhorn master as #{respawn_commandline.inspect}", :reload)
         Einhorn::Compat.exec(*respawn_commandline)
       rescue SystemCallError => e
@@ -260,30 +270,34 @@ module Einhorn
       end
     end
 
-    def self.spinup(cmd=nil)
+    def self.spinup(cmd = nil)
       cmd ||= Einhorn::State.cmd
       index = next_index
-      if Einhorn::TransientState.preloaded
-        pid = fork do
+      expected_ppid = Process.pid
+      pid = if Einhorn::State.preloaded
+        fork do
           Einhorn::TransientState.whatami = :worker
           prepare_child_process
 
-          Einhorn.log_info('About to tear down Einhorn state and run einhorn_main')
+          Einhorn.log_info("About to tear down Einhorn state and run einhorn_main")
           Einhorn::Command::Interface.uninit
           Einhorn::Event.close_all_for_worker
           Einhorn.set_argv(cmd, true)
 
           reseed_random
 
+          setup_parent_watch(expected_ppid)
+
           prepare_child_environment(index)
           einhorn_main
         end
       else
-        pid = fork do
+        fork do
           Einhorn::TransientState.whatami = :worker
           prepare_child_process
 
           Einhorn.log_info("About to exec #{cmd.inspect}")
+          Einhorn::Command::Interface.uninit
           # Here's the only case where cloexec would help. Since we
           # have to track and manually close FDs for other cases, we
           # may as well just reuse close_all rather than also set
@@ -292,20 +306,23 @@ module Einhorn
           # Note that Ruby 1.9's close_others option is useful here.
           Einhorn::Event.close_all_for_worker
 
+          setup_parent_watch(expected_ppid)
+
           prepare_child_environment(index)
-          Einhorn::Compat.exec(cmd[0], cmd[1..-1], :close_others => false)
+          Einhorn::Compat.exec(cmd[0], cmd[1..-1], close_others: false)
         end
       end
 
       Einhorn.log_info("===> Launched #{pid} (index: #{index})", :upgrade)
       Einhorn::State.last_spinup = Time.now
       Einhorn::State.children[pid] = {
-        :type => :worker,
-        :version => Einhorn::State.version,
-        :acked => false,
-        :signaled => Set.new,
-        :index => index,
-        :spinup_time => Einhorn::State.last_spinup,
+        type: :worker,
+        version: Einhorn::State.version,
+        acked: false,
+        signaled: Set.new,
+        last_signaled_at: nil,
+        index: index,
+        spinup_time: Einhorn::State.last_spinup
       }
 
       # Set up whatever's needed for ACKing
@@ -314,6 +331,7 @@ module Einhorn
       when :timer
         Einhorn::Event::ACKTimer.open(ack_mode[:timeout], pid)
       when :manual
+        # nothing to do
       else
         Einhorn.log_error("Unrecognized ACK mode #{type.inspect}")
       end
@@ -321,24 +339,18 @@ module Einhorn
 
     def self.prepare_child_environment(index)
       # This is run from the child
-      ENV['EINHORN_MASTER_PID'] = Process.ppid.to_s
-      ENV['EINHORN_SOCK_PATH'] = Einhorn::Command::Interface.socket_path
+      ENV["EINHORN_MASTER_PID"] = Process.ppid.to_s
+      ENV["EINHORN_SOCK_PATH"] = Einhorn::Command::Interface.socket_path
       if Einhorn::State.command_socket_as_fd
         socket = UNIXSocket.open(Einhorn::Command::Interface.socket_path)
         Einhorn::TransientState.socket_handles << socket
-        ENV['EINHORN_SOCK_FD'] = socket.fileno.to_s
+        ENV["EINHORN_SOCK_FD"] = socket.fileno.to_s
       end
 
-      ENV['EINHORN_FD_COUNT'] = Einhorn::State.bind_fds.length.to_s
-      Einhorn::State.bind_fds.each_with_index {|fd, i| ENV["EINHORN_FD_#{i}"] = fd.to_s}
-
-      ENV['EINHORN_CHILD_INDEX'] = index.to_s
+      ENV["EINHORN_FD_COUNT"] = Einhorn::State.bind_fds.length.to_s
+      Einhorn::State.bind_fds.each_with_index { |fd, i| ENV["EINHORN_FD_#{i}"] = fd.to_s }
 
-      # EINHORN_FDS is deprecated. It was originally an attempt to
-      # match Upstart's nominal internal support for space-separated
-      # FD lists, but nobody uses that in practice, and it makes
-      # finding individual FDs more difficult
-      ENV['EINHORN_FDS'] = Einhorn::State.bind_fds.map(&:to_s).join(' ')
+      ENV["EINHORN_CHILD_INDEX"] = index.to_s
     end
 
     # Reseed common ruby random number generators.
@@ -361,11 +373,11 @@ module Einhorn
 
       # reseed OpenSSL::Random if it's loaded
       if defined?(OpenSSL::Random)
-        if defined?(Random)
-          seed = Random.new_seed
+        seed = if defined?(Random)
+          Random.new_seed
         else
           # Ruby 1.8
-          seed = rand
+          rand
         end
         OpenSSL::Random.seed(seed.to_s)
       end
@@ -376,6 +388,24 @@ module Einhorn
       Einhorn.renice_self
     end
 
+    def self.setup_parent_watch(expected_ppid)
+      if Einhorn::State.kill_children_on_exit
+        begin
+          # NB: Having the USR2 signal handler set to terminate (the default) at
+          # this point is required. If it's set to a ruby handler, there are
+          # race conditions that could cause the worker to leak.
+
+          Einhorn::Prctl.set_pdeathsig("USR2")
+          if Process.ppid != expected_ppid
+            Einhorn.log_error("Parent process died before we set pdeathsig; cowardly refusing to exec child process.")
+            exit(1)
+          end
+        rescue NotImplementedError
+          # Unsupported OS; silently continue.
+        end
+      end
+    end
+
     # @param options [Hash]
     #
     # @option options [Boolean] :smooth (false) Whether to perform a smooth or
@@ -384,18 +414,19 @@ module Einhorn
     #   upgrade, bring up all the new workers and don't cull any old workers
     #   until they're all up.
     #
-    def self.full_upgrade(options={})
-      options = {:smooth => false}.merge(options)
+    def self.full_upgrade(options = {})
+      options = {smooth: false}.merge(options)
 
       Einhorn::State.smooth_upgrade = options.fetch(:smooth)
       reload_for_upgrade
     end
 
     def self.full_upgrade_smooth
-      full_upgrade(:smooth => true)
+      full_upgrade(smooth: true)
     end
+
     def self.full_upgrade_fleet
-      full_upgrade(:smooth => false)
+      full_upgrade(smooth: false)
     end
 
     def self.reload_for_upgrade
@@ -408,8 +439,8 @@ module Einhorn
         Einhorn.log_info("Currently upgrading (#{Einhorn::WorkerPool.ack_count} / #{Einhorn::WorkerPool.ack_target} ACKs; bumping version and starting over)...", :upgrade)
       else
         Einhorn::State.upgrading = true
-        u_type = Einhorn::State.smooth_upgrade ? 'smooth' : 'fleet'
-        Einhorn.log_info("Starting #{u_type} upgrade from version" +
+        u_type = Einhorn::State.smooth_upgrade ? "smooth" : "fleet"
+        Einhorn.log_info("Starting #{u_type} upgrade from version" \
                          " #{Einhorn::State.version}...", :upgrade)
       end
 
@@ -456,10 +487,45 @@ module Einhorn
       end
 
       if unsignaled > target
-        excess = Einhorn::WorkerPool.unsignaled_modern_workers_with_priority[0...(unsignaled-target)]
+        excess = Einhorn::WorkerPool.unsignaled_modern_workers_with_priority[0...(unsignaled - target)]
         Einhorn.log_info("Have too many workers at the current version, so killing off #{excess.length} of them.")
         signal_all("USR2", excess)
       end
+
+      # Ensure all signaled workers that have outlived signal_timeout get killed.
+      kill_expired_signaled_workers if Einhorn::State.signal_timeout
+    end
+
+    def self.kill_expired_signaled_workers
+      now = Time.now
+      children = Einhorn::State.children.select do |_, c|
+        # Only interested in USR2 signaled workers
+        next unless c[:signaled] && c[:signaled].length > 0
+        next unless c[:signaled].include?("USR2")
+
+        # Ignore processes that have received KILL since it can't be trapped.
+        next if c[:signaled].include?("KILL")
+
+        # Filter out those children that have not reached signal_timeout yet.
+        next unless c[:last_signaled_at]
+        expires_at = c[:last_signaled_at] + Einhorn::State.signal_timeout
+        next unless now >= expires_at
+
+        true
+      end
+
+      Einhorn.log_info("#{children.size} expired signaled workers found.") if children.size > 0
+      children.each do |pid, child|
+        Einhorn.log_info("Child #{pid.inspect} was signaled #{(child[:last_signaled_at] - now).abs.to_i}s ago. Sending SIGKILL as it is still active after #{Einhorn::State.signal_timeout}s timeout.", :upgrade)
+        begin
+          Process.kill("KILL", pid)
+        rescue Errno::ESRCH
+          Einhorn.log_debug("Attempted to SIGKILL child #{pid.inspect} but the process does not exist.")
+        end
+
+        child[:signaled].add("KILL")
+        child[:last_signaled_at] = Time.now
+      end
     end
 
     def self.stop_respawning
@@ -484,7 +550,7 @@ module Einhorn
         return
       end
       Einhorn.log_info("Launching #{missing} new workers")
-      missing.times {spinup}
+      missing.times { spinup }
     end
 
     # Unbounded exponential backoff is not a thing: we run into problems if
@@ -493,10 +559,12 @@ module Einhorn
     # don't wait until the heat death of the universe to spin up new capacity.
     MAX_SPINUP_INTERVAL = 30.0
 
-    def self.replenish_gradually(max_unacked=nil)
+    def self.replenish_gradually(max_unacked = nil)
       return if Einhorn::TransientState.has_outstanding_spinup_timer
       return unless Einhorn::WorkerPool.missing_worker_count > 0
 
+      max_unacked ||= Einhorn::State.config[:max_unacked]
+
       # default to spinning up at most NCPU workers at once
       unless max_unacked
         begin
@@ -514,16 +582,13 @@ module Einhorn
 
       # Exponentially backoff automated spinup if we're just having
       # things die before ACKing
-      spinup_interval = Einhorn::State.config[:seconds] * (1.5 ** Einhorn::State.consecutive_deaths_before_ack)
+      spinup_interval = Einhorn::State.config[:seconds] * (1.5**Einhorn::State.consecutive_deaths_before_ack)
       spinup_interval = [spinup_interval, MAX_SPINUP_INTERVAL].min
       seconds_ago = (Time.now - Einhorn::State.last_spinup).to_f
 
       if seconds_ago > spinup_interval
-        unacked = Einhorn::WorkerPool.unacked_unsignaled_modern_workers.length
-        if unacked >= max_unacked
-          Einhorn.log_debug("There are #{unacked} unacked new workers, and max_unacked is #{max_unacked}, so not spinning up a new process")
-        else
-          msg = "Last spinup was #{seconds_ago}s ago, and spinup_interval is #{spinup_interval}s, so spinning up a new process"
+        if trigger_spinup?(max_unacked)
+          msg = "Last spinup was #{seconds_ago}s ago, and spinup_interval is #{spinup_interval}s, so spinning up a new process."
 
           if Einhorn::State.consecutive_deaths_before_ack > 0
             Einhorn.log_info("#{msg} (there have been #{Einhorn::State.consecutive_deaths_before_ack} consecutive unacked worker deaths)", :upgrade)
@@ -534,7 +599,7 @@ module Einhorn
           spinup
         end
       else
-        Einhorn.log_debug("Last spinup was #{seconds_ago}s ago, and spinup_interval is #{spinup_interval}s, so not spinning up a new process")
+        Einhorn.log_debug("Last spinup was #{seconds_ago}s ago, and spinup_interval is #{spinup_interval}s, so not spinning up a new process.")
       end
 
       Einhorn::TransientState.has_outstanding_spinup_timer = true
@@ -544,18 +609,35 @@ module Einhorn
       end
     end
 
-    def self.quieter(log=true)
+    def self.quieter(log = true)
       Einhorn::State.verbosity += 1 if Einhorn::State.verbosity < 2
       output = "Verbosity set to #{Einhorn::State.verbosity}"
       Einhorn.log_info(output) if log
       output
     end
 
-    def self.louder(log=true)
+    def self.louder(log = true)
       Einhorn::State.verbosity -= 1 if Einhorn::State.verbosity > 0
       output = "Verbosity set to #{Einhorn::State.verbosity}"
       Einhorn.log_info(output) if log
       output
     end
+
+    def self.trigger_spinup?(max_unacked)
+      unacked = Einhorn::WorkerPool.unacked_unsignaled_modern_workers.length
+      if unacked >= max_unacked
+        Einhorn.log_info("There are #{unacked} unacked new workers, and max_unacked is #{max_unacked}, so not spinning up a new process.")
+        return false
+      elsif Einhorn::State.config[:max_upgrade_additional]
+        capacity_exceeded = (Einhorn::State.config[:number] + Einhorn::State.config[:max_upgrade_additional]) - Einhorn::WorkerPool.workers_with_state.length
+        if capacity_exceeded < 0
+          Einhorn.log_info("Over worker capacity by #{capacity_exceeded.abs} during upgrade, #{Einhorn::WorkerPool.modern_workers.length} new workers of #{Einhorn::WorkerPool.workers_with_state.length} total. Waiting for old workers to exit before spinning up a process.")
+
+          return false
+        end
+      end
+
+      true
+    end
   end
 end

data/lib/einhorn/compat.rb

@@ -11,10 +11,10 @@ module Einhorn
 
     def self.cloexec!(fd, enable)
       original = fd.fcntl(Fcntl::F_GETFD)
-      if enable
-        new = original | Fcntl::FD_CLOEXEC
+      new = if enable
+        original | Fcntl::FD_CLOEXEC
       else
-        new = original & (-Fcntl::FD_CLOEXEC-1)
+        original & (-Fcntl::FD_CLOEXEC - 1)
       end
       fd.fcntl(Fcntl::F_SETFD, new)
     end
@@ -24,7 +24,7 @@ module Einhorn
     end
 
     # Opts are ignored in Ruby 1.8
-    def self.exec(script, args, opts={})
+    def self.exec(script, args, opts = {})
       cmd = [script, script]
       begin
         Kernel.exec(cmd, *(args + [opts]))
@@ -53,18 +53,18 @@ module Einhorn
 
       # linux / friends
       begin
-        return File.read('/proc/cpuinfo').scan(/^processor\s*:/).count
+        return File.read("/proc/cpuinfo").scan(/^processor\s*:/).count
       rescue Errno::ENOENT
       end
 
       # OS X
-      if RUBY_PLATFORM =~ /darwin/
+      if RUBY_PLATFORM.match?(/darwin/)
         return Integer(`sysctl -n hw.logicalcpu`)
       end
 
       # windows / friends
       begin
-        require 'win32ole'
+        require "win32ole"
       rescue LoadError
       else
         wmi = WIN32OLE.connect("winmgmts://")