einhorn 0.5.4 → 0.5.5

data/einhorn.gemspec

@@ -14,13 +14,12 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = 'einhorn'
   gem.require_paths = ['lib']
+
   gem.add_development_dependency 'rake'
-  gem.add_development_dependency 'minitest'
-  gem.add_development_dependency 'mocha'
+  gem.add_development_dependency 'pry'
+  gem.add_development_dependency 'minitest', '< 5.0'
+  gem.add_development_dependency 'mocha', '~> 0.13'
   gem.add_development_dependency 'chalk-rake'
 
-  gem.add_development_dependency('rake')
-  gem.add_development_dependency('minitest', '< 5.0')
-  gem.add_development_dependency('mocha', '~> 0.13')
   gem.version       = Einhorn::VERSION
 end

data/example/time_server

@@ -15,18 +15,36 @@
 require 'rubygems'
 require 'einhorn/worker'
 
+def log(msg)
+  puts "=== [#{$$}] #{msg}"
+end
+
 def einhorn_main
-  puts "Called with ENV['EINHORN_FD_0']: #{ENV['EINHORN_FD_0']}"
+  log "Called with ENV['EINHORN_FD_0']: #{ENV['EINHORN_FD_0']}"
 
   fd_num = Einhorn::Worker.socket!
   socket = Socket.for_fd(fd_num)
 
+  sleep_before_shutdown = 0
+  sleep_before_ack = 0
+
   # Came up successfully, so let's set up graceful handler and ACK the
   # master.
   Einhorn::Worker.graceful_shutdown do
-    puts "Goodbye from #{$$}"
+    if sleep_before_shutdown > 0
+      log "sleeping #{sleep_before_shutdown}s before shutdown"
+      sleep sleep_before_shutdown
+    end
+    log "Goodbye!"
     exit(0)
   end
+
+  if sleep_before_ack > 0
+    log "sleeping #{sleep_before_ack}s before ack"
+    sleep sleep_before_ack
+  end
+  log "worker ack"
+
   Einhorn::Worker.ack!
 
   # Real work happens here.

data/lib/einhorn/command/interface.rb

@@ -162,10 +162,10 @@ module Einhorn::Command
         Einhorn::Command.stop_respawning
         exit(1)
       end
-      trap_async("HUP") {Einhorn::Command.full_upgrade}
+      trap_async("HUP") {Einhorn::Command.full_upgrade_smooth}
       trap_async("ALRM") do
         Einhorn.log_error("Upgrading using SIGALRM is deprecated. Please switch to SIGHUP")
-        Einhorn::Command.full_upgrade
+        Einhorn::Command.full_upgrade_smooth
       end
       trap_async("CHLD") {}
       trap_async("USR2") do
@@ -292,7 +292,8 @@ EOF
     # Used by einhornsh
     command 'ehlo' do |conn, request|
       <<EOF
-Welcome #{request['user']}! You are speaking to Einhorn Master Process #{$$}#{Einhorn::State.cmd_name ? " (#{Einhorn::State.cmd_name})" : ''}
+Welcome, #{request['user']}! You are speaking to Einhorn Master Process #{$$}#{Einhorn::State.cmd_name ? " (#{Einhorn::State.cmd_name})" : ''}.
+This is Einhorn #{Einhorn::VERSION}.
 EOF
     end
 
@@ -335,14 +336,25 @@ EOF
       Einhorn::Command.louder
     end
 
-    command 'upgrade', 'Upgrade all Einhorn workers. This may result in Einhorn reloading its own code as well.' do |conn, request|
+    command 'upgrade', 'Upgrade all Einhorn workers smoothly. This may result in Einhorn reloading its own code as well.' do |conn, request|
       # send first message directly for old clients that don't support request
       # ids or subscriptions. Everything else is sent tagged with request id
       # for new clients.
-      send_message(conn, 'Upgrading, as commanded', request['id'])
+      send_message(conn, 'Upgrading smoothly, as commanded', request['id'])
       conn.subscribe(:upgrade, request['id'])
       # If the app is preloaded this doesn't return.
-      Einhorn::Command.full_upgrade
+      Einhorn::Command.full_upgrade_smooth
+      nil
+    end
+
+    command 'upgrade_fleet', 'Upgrade all Einhorn workers a fleet at a time. This may result in Einhorn reloading its own code as well.' do |conn, request|
+      # send first message directly for old clients that don't support request
+      # ids or subscriptions. Everything else is sent tagged with request id
+      # for new clients.
+      send_message(conn, 'Upgrading fleet, as commanded', request['id'])
+      conn.subscribe(:upgrade, request['id'])
+      # If the app is preloaded this doesn't return.
+      Einhorn::Command.full_upgrade_fleet
       nil
     end
 

data/lib/einhorn/command.rb

@@ -232,6 +232,8 @@ module Einhorn
           Einhorn::Event.close_all_for_worker
           Einhorn.set_argv(cmd, true)
 
+          reseed_random
+
           prepare_child_environment
           einhorn_main
         end
@@ -294,11 +296,53 @@ module Einhorn
       ENV['EINHORN_FDS'] = Einhorn::State.bind_fds.map(&:to_s).join(' ')
     end
 
+    # Reseed common ruby random number generators.
+    #
+    # OpenSSL::Random uses the PID to reseed after fork, which means that if a
+    # long-lived master process over its lifetime spawns two workers with the
+    # same PID, those workers will start with the same OpenSSL seed.
+    #
+    # Ruby >= 1.9 has a guard against this in SecureRandom, but any direct
+    # users of OpenSSL::Random will still be affected.
+    #
+    # Ruby 1.8 didn't even reseed the default random number generator used by
+    # Kernel#rand in certain releases.
+    #
+    # https://bugs.ruby-lang.org/issues/4579
+    #
+    def self.reseed_random
+      # reseed Kernel#rand
+      srand
+
+      # reseed OpenSSL::Random if it's loaded
+      if defined?(OpenSSL::Random)
+        if defined?(Random)
+          seed = Random.new_seed
+        else
+          # Ruby 1.8
+          seed = rand
+        end
+        OpenSSL::Random.seed(seed.to_s)
+      end
+    end
+
     def self.prepare_child_process
       Einhorn.renice_self
     end
 
-    def self.full_upgrade
+    # @param options [Hash]
+    #
+    # @option options [Boolean] :smooth (false) Whether to perform a smooth or
+    #   fleet upgrade. In a smooth upgrade, bring up new workers and cull old
+    #   workers one by one as soon as there is a replacement. In a fleet
+    #   upgrade, bring up all the new workers and don't cull any old workers
+    #   until they're all up.
+    #
+    def self.full_upgrade(options={})
+      options = {:smooth => false}.merge(options)
+
+      Einhorn::State.smooth_upgrade = options.fetch(:smooth)
+
       if Einhorn::State.path && !Einhorn::State.reloading_for_preload_upgrade
         reload_for_preload_upgrade
       else
@@ -306,6 +350,13 @@ module Einhorn
       end
     end
 
+    def self.full_upgrade_smooth
+      full_upgrade(:smooth => true)
+    end
+    def self.full_upgrade_fleet
+      full_upgrade(:smooth => false)
+    end
+
     def self.reload_for_preload_upgrade
       Einhorn::State.reloading_for_preload_upgrade = true
       reload
@@ -316,7 +367,9 @@ module Einhorn
         Einhorn.log_info("Currently upgrading (#{Einhorn::WorkerPool.ack_count} / #{Einhorn::WorkerPool.ack_target} ACKs; bumping version and starting over)...", :upgrade)
       else
         Einhorn::State.upgrading = true
-        Einhorn.log_info("Starting upgrade from version #{Einhorn::State.version}...", :upgrade)
+        u_type = Einhorn::State.smooth_upgrade ? 'smooth' : 'fleet'
+        Einhorn.log_info("Starting #{u_type} upgrade from version" +
+                         " #{Einhorn::State.version}...", :upgrade)
       end
 
       # Reset this, since we've just upgraded to a new universe (I'm
@@ -326,7 +379,11 @@ module Einhorn
       Einhorn::State.last_upgraded = Time.now
 
       Einhorn::State.version += 1
-      replenish_immediately
+      if Einhorn::State.smooth_upgrade
+        replenish_gradually
+      else
+        replenish_immediately
+      end
     end
 
     def self.cull
@@ -341,9 +398,20 @@ module Einhorn
       end
 
       old_workers = Einhorn::WorkerPool.old_workers
+      Einhorn.log_debug("#{acked} acked, #{unsignaled} unsignaled, #{target} target, #{old_workers.length} old workers")
       if !Einhorn::State.upgrading && old_workers.length > 0
         Einhorn.log_info("Killing off #{old_workers.length} old workers.")
         signal_all("USR2", old_workers)
+      elsif Einhorn::State.upgrading && Einhorn::State.smooth_upgrade
+        # In a smooth upgrade, kill off old workers one by one when we have
+        # sufficiently many new workers.
+        excess = (old_workers.length + acked) - target
+        if excess > 0
+          Einhorn.log_info("Smooth upgrade: killing off #{excess} old workers.")
+          signal_all("USR2", old_workers.take(excess))
+        elsif excess < 0
+          Einhorn.log_error("Smooth upgrade: somehow excess is #{excess}!")
+        end
       end
 
       if unsignaled > target
@@ -378,25 +446,45 @@ module Einhorn
       missing.times {spinup}
     end
 
-    def self.replenish_gradually
+    def self.replenish_gradually(max_unacked=nil)
       return if Einhorn::TransientState.has_outstanding_spinup_timer
       return unless Einhorn::WorkerPool.missing_worker_count > 0
 
+      # default to spinning up at most NCPU workers at once
+      unless max_unacked
+        begin
+          @processor_count ||= Einhorn::Compat.processor_count
+        rescue => err
+          Einhorn.log_error(err.inspect)
+          @processor_count = 1
+        end
+        max_unacked = @processor_count
+      end
+
+      if max_unacked <= 0
+        raise ArgumentError.new("max_unacked must be positive")
+      end
+
       # Exponentially backoff automated spinup if we're just having
       # things die before ACKing
       spinup_interval = Einhorn::State.config[:seconds] * (1.5 ** Einhorn::State.consecutive_deaths_before_ack)
       seconds_ago = (Time.now - Einhorn::State.last_spinup).to_f
 
       if seconds_ago > spinup_interval
-        msg = "Last spinup was #{seconds_ago}s ago, and spinup_interval is #{spinup_interval}s, so spinning up a new process"
-
-        if Einhorn::State.consecutive_deaths_before_ack > 0
-          Einhorn.log_info("#{msg} (there have been #{Einhorn::State.consecutive_deaths_before_ack} consecutive unacked worker deaths)")
+        unacked = Einhorn::WorkerPool.unacked_unsignaled_modern_workers.length
+        if unacked >= max_unacked
+          Einhorn.log_debug("There are #{unacked} unacked new workers, and max_unacked is #{max_unacked}, so not spinning up a new process")
         else
-          Einhorn.log_debug(msg)
-        end
+          msg = "Last spinup was #{seconds_ago}s ago, and spinup_interval is #{spinup_interval}s, so spinning up a new process"
 
-        spinup
+          if Einhorn::State.consecutive_deaths_before_ack > 0
+            Einhorn.log_info("#{msg} (there have been #{Einhorn::State.consecutive_deaths_before_ack} consecutive unacked worker deaths)")
+          else
+            Einhorn.log_debug(msg)
+          end
+
+          spinup
+        end
       else
         Einhorn.log_debug("Last spinup was #{seconds_ago}s ago, and spinup_interval is #{spinup_interval}s, so not spinning up a new process")
       end

data/lib/einhorn/compat.rb

@@ -44,5 +44,41 @@ module Einhorn
       cloexec!(sock, false)
       sock
     end
+
+    def self.processor_count
+      # jruby
+      if defined? Java::Java
+        return Java::Java.lang.Runtime.getRuntime.availableProcessors
+      end
+
+      # linux / friends
+      begin
+        return File.read('/proc/cpuinfo').scan(/^processor\s*:/).count
+      rescue Errno::ENOENT
+      end
+
+      # OS X
+      if RUBY_PLATFORM =~ /darwin/
+        return Integer(`sysctl -n hw.logicalcpu`)
+      end
+
+      # windows / friends
+      begin
+        require 'win32ole'
+      rescue LoadError
+      else
+        wmi = WIN32OLE.connect("winmgmts://")
+        wmi.ExecQuery("select * from Win32_ComputerSystem").each do |system|
+          begin
+            processors = system.NumberOfLogicalProcessors
+          rescue
+            processors = 0
+          end
+          return [system.NumberOfProcessors, processors].max
+        end
+      end
+
+      raise "Failed to detect number of CPUs"
+    end
   end
 end

data/lib/einhorn/version.rb

@@ -1,3 +1,3 @@
 module Einhorn
-  VERSION = '0.5.4'
+  VERSION = '0.5.5'
 end

data/lib/einhorn/worker_pool.rb

@@ -52,6 +52,16 @@ module Einhorn
       end.map {|pid, _| pid}
     end
 
+    def self.unacked_unsignaled_modern_workers_with_state
+      modern_workers_with_state.select {|pid, spec|
+        !spec[:acked] && spec[:signaled].length == 0
+      }
+    end
+
+    def self.unacked_unsignaled_modern_workers
+      unacked_unsignaled_modern_workers_with_state.map {|pid, _| pid}
+    end
+
     # Use the number of modern workers, rather than unsignaled modern
     # workers. This means if e.g. we do bunch of decs and then incs,
     # any workers which haven't died yet will count towards our number

data/lib/einhorn.rb

@@ -55,6 +55,7 @@ module Einhorn
         :script_name => nil,
         :respawn => true,
         :upgrading => false,
+        :smooth_upgrade => false,
         :reloading_for_preload_upgrade => false,
         :path => nil,
         :cmd_name => nil,

data/test/unit/einhorn/command/interface.rb

@@ -12,7 +12,7 @@ class InterfaceTest < EinhornTestCase
         # Remove trailing newline
         message = message[0...-1]
         parsed = YAML.load(URI.unescape(message))
-        parsed['message'] =~ /Welcome gdb/
+        parsed['message'] =~ /Welcome, gdb/
       end
       request = {
         'command' => 'ehlo',

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: einhorn
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.5.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-11 00:00:00.000000000 Z
+date: 2014-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -28,7 +28,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -44,39 +44,39 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: mocha
+  name: minitest
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - <
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - <
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: chalk-rake
+  name: mocha
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.13'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: chalk-rake
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -91,38 +91,6 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - <
-      - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - <
-      - !ruby/object:Gem::Version
-        version: '5.0'
-- !ruby/object:Gem::Dependency
-  name: mocha
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.13'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '0.13'
 description: Einhorn makes it easy to run multiple instances of an application server,
   all listening on the same port. You can also seamlessly restart your workers without
   dropping any requests. Einhorn requires minimal application-level support, making
@@ -198,7 +166,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4111733587865506244
+      hash: -690549708583071819
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -207,7 +175,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -4111733587865506244
+      hash: -690549708583071819
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23