ultravisor 0.0.0.3.g8cf10dc

data/lib/ultravisor/child.rb

@@ -0,0 +1,481 @@
+# frozen_string_literal: true
+
+require_relative "./logging_helpers"
+
+class Ultravisor
+	class Child
+		include LoggingHelpers
+
+		attr_reader :id
+
+		def initialize(id:,
+		               klass:,
+		               args: [],
+		               method:,
+		               restart: :always,
+		               restart_policy: {
+		                  period: 5,
+		                  max: 3,
+		                  delay: 1,
+		               },
+		               shutdown: {
+		                  method: nil,
+		                  timeout: 1,
+		               },
+		               logger: Logger.new("/dev/null"),
+		               enable_castcall: false,
+		               access: nil
+		              )
+
+			@logger = logger
+			@id = id
+
+			@klass, @args, @method = klass, args, method
+			validate_kam
+
+			@restart = restart
+			validate_restart
+
+			@restart_policy = restart_policy
+			validate_restart_policy
+
+			@shutdown_spec = shutdown
+			validate_shutdown_spec
+
+			@access = access
+			validate_access
+
+			@enable_castcall = enable_castcall
+
+			@runtime_history = []
+
+			@spawn_m    = Mutex.new
+			@spawn_cv   = ConditionVariable.new
+
+			@shutdown_m = Mutex.new
+		end
+
+		def spawn(term_queue)
+			@spawn_m.synchronize do
+				@value      = nil
+				@exception  = nil
+				@start_time = Time.now
+				@instance   = new_instance
+
+				@spawn_id = sid = rand
+
+				Thread.handle_interrupt(::Exception => :never, ::Numeric => :never) do
+					@thread = Thread.new do
+						Thread.current.name = "Ultravisor::Child(#{@id})"
+						logger.debug(logloc) { "Spawning new instance of #{@id}" }
+
+						begin
+							Thread.handle_interrupt(::Exception => :immediate, ::Numeric => :immediate) do
+								logger.debug(logloc) { "Calling #{@klass}##{@method} to start #{@id} running" }
+								@value = @instance.public_send(@method)
+							end
+						rescue Exception => ex
+							@exception = ex
+						ensure
+							@spawn_m.synchronize do
+								# Even if a thread gets whacked by Thread#kill, ensure blocks
+								# still get run.  This is... wonderful!  And terrifying!
+
+								termination_cleanup(term_queue) if @spawn_id == sid
+							end
+						end
+					end
+				end
+
+				@spawn_cv.broadcast
+			end
+
+			self
+		end
+
+		def shutdown(force: false)
+			@shutdown_m.synchronize do
+				th  = nil
+				sid = nil
+
+				@spawn_m.synchronize do
+					return if @thread.nil? || @thread == Thread.current
+
+					# Take a reference to the running thread, so we don't need to
+					# keep acquiring spawn_m every time we want to do something
+					# with it -- which causes collisions when it comes time to
+					# wait on the terminating thread, which is itself is trying
+					# to acquire the same lock so it can cleanup.
+					th  = @thread
+					sid = @spawn_id
+
+					# Let everyone know we're in shutdown mode
+					@shutting_down = true
+				end
+
+				if @shutdown_spec[:method] && !force
+					begin
+						@instance.public_send(@shutdown_spec[:method])
+					rescue Exception => ex
+						log_exception(ex) { "Unhandled exception when calling #{@shutdown_spec[:method].inspect} on child #{id}" }
+						th.kill
+					end
+				else
+					th.kill
+				end
+
+				unless th.join(@shutdown_spec[:timeout])
+					logger.info(logloc) { "Child instance for #{self.id} did not cleanly shutdown within #{@shutdown_spec[:timeout]} seconds; force-killing the thread" }
+					th.kill
+				end
+
+				# Last chance, bubs
+				unless th.join(0.1)
+					logger.error(logloc) { "Child thread for #{self.id} appears hung; abandoning thread #{th}" }
+
+					# If we get here, then the worker instance has seized up spectacularly,
+					# and the cleanup in the `spawn` ensure hasn't triggered, so we need
+					# to do the cleanup instead.
+					@spawn_m.synchronize do
+						termination_cleanup if @spawn_id == sid
+					end
+				end
+			end
+		end
+
+		def wait
+			@spawn_m.synchronize do
+				@spawn_cv.wait(@spawn_m) while @thread
+			end
+		end
+
+		def termination_exception
+			@spawn_m.synchronize do
+				@spawn_cv.wait(@spawn_m) while @thread
+				@exception
+			end
+		end
+
+		def termination_value
+			@spawn_m.synchronize do
+				@spawn_cv.wait(@spawn_m) while @thread
+				@value
+			end
+		end
+
+		def restart_delay
+			d = begin
+				case @restart_policy[:delay]
+				when Numeric
+					@restart_policy[:delay]
+				when Range
+					@restart_policy[:delay].first + (@restart_policy[:delay].last - @restart_policy[:delay].first) * rand
+				end
+			end
+
+			[0, d].max
+		end
+
+		def restart?
+			if blown_policy?
+				raise BlownRestartPolicyError,
+				      "Child #{self.id} has restarted more than #{@restart_policy[:max]} times in #{@restart_policy[:period]} seconds."
+			end
+
+			!!(@restart == :always || (@restart == :on_failure && termination_exception))
+		end
+
+		def unsafe_instance
+			unless @access == :unsafe
+				raise Ultravisor::ThreadSafetyError,
+				      "#unsafe_instance called on a child not declared with access: :unsafe"
+			end
+
+			current_instance
+		end
+
+		def cast
+			unless castcall_enabled?
+				raise NoMethodError,
+				      "undefined method `cast' for #{self}"
+			end
+
+			CastReceiver.new do |castback|
+				@spawn_m.synchronize do
+					while @instance.nil?
+						#:nocov:
+						@spawn_cv.wait(@spawn_m)
+						#:nocov:
+					end
+
+					unless @instance.respond_to? castback.method_name
+						raise NoMethodError,
+						      "undefined method `#{castback.method_name}' for #{@instance}"
+					end
+
+					begin
+						@instance.instance_variable_get(:@ultravisor_child_castcall_queue) << castback
+					rescue ClosedQueueError
+						# casts aren't guaranteed to ever execute, so dropping it
+						# when the instance's queue has closed is perfectly valid
+					end
+
+					@castcall_fd_writer.putc "!"
+				end
+			end
+		end
+
+		def call
+			unless castcall_enabled?
+				raise NoMethodError,
+				      "undefined method `call' for #{self}"
+			end
+
+			CallReceiver.new do |callback|
+				@spawn_m.synchronize do
+					while @instance.nil?
+						#:nocov:
+						@spawn_cv.wait(@spawn_m)
+						#:nocov:
+					end
+
+					unless @instance.respond_to? callback.method_name
+						raise NoMethodError,
+						      "undefined method `#{callback.method_name}' for #{@instance}"
+					end
+
+					begin
+						@instance.instance_variable_get(:@ultravisor_child_castcall_queue) << callback
+					rescue ClosedQueueError
+						raise ChildRestartedError
+					end
+
+					@castcall_fd_writer.putc "!"
+				end
+			end
+		end
+
+		private
+
+		def validate_kam
+			if @klass.instance_method(:initialize).arity == 0 && @args != []
+				raise InvalidKAMError,
+				      "#{@klass.to_s}.new takes no arguments, but args not empty."
+			end
+
+			begin
+				if @klass.instance_method(@method).arity != 0
+					raise InvalidKAMError,
+					      "#{@klass.to_s}##{@method} must not take arguments"
+				end
+			rescue NameError
+				raise InvalidKAMError,
+				      "#{@klass.to_s} has no instance method #{@method}"
+			end
+		end
+
+		def validate_restart
+			unless %i{never on_failure always}.include?(@restart)
+				raise ArgumentError,
+				      "Invalid value for restart: #{@restart.inspect}"
+			end
+		end
+
+		def validate_restart_policy
+			unless @restart_policy.is_a?(Hash)
+				raise ArgumentError,
+				      "restart_policy must be a hash (got #{@restart_policy.inspect})"
+			end
+
+			bad_keys = @restart_policy.keys - %i{period max delay}
+			unless bad_keys.empty?
+				raise ArgumentError,
+				      "Invalid key(s) in restart_policy: #{bad_keys.inspect}"
+			end
+
+			# Restore any missing defaults
+			@restart_policy = { period: 5, max: 3, delay: 1 }.merge(@restart_policy)
+
+			unless @restart_policy[:period].is_a?(Numeric) && @restart_policy[:period].positive?
+				raise ArgumentError,
+				      "Invalid restart_policy period #{@restart_policy[:period].inspect}: must be positive integer"
+			end
+
+			unless @restart_policy[:max].is_a?(Numeric) && !@restart_policy[:max].negative?
+				raise ArgumentError,
+				      "Invalid restart_policy max #{@restart_policy[:period].inspect}: must be non-negative integer"
+			end
+
+			case @restart_policy[:delay]
+			when Numeric
+				if @restart_policy[:delay].negative?
+					raise ArgumentError,
+					      "Invalid restart_policy delay #{@restart_policy[:delay].inspect}: must be non-negative integer or range"
+				end
+			when Range
+				if @restart_policy[:delay].first >= @restart_policy[:delay].last
+					raise ArgumentError,
+					      "Invalid restart_policy delay #{@restart_policy[:delay].inspect}: must be non-negative integer or increasing range"
+				end
+
+				if @restart_policy[:delay].first.negative?
+					raise ArgumentError,
+					      "Invalid restart_policy delay #{@restart_policy[:delay].inspect}: range must not be negative"
+				end
+			else
+				raise ArgumentError,
+				      "Invalid restart_policy delay #{@restart_policy[:delay].inspect}: must be non-negative integer or range"
+			end
+		end
+
+		def validate_shutdown_spec
+			unless @shutdown_spec.is_a?(Hash)
+				raise ArgumentError,
+				      "shutdown must be a hash (got #{@shutdown_spec.inspect})"
+			end
+
+			bad_keys = @shutdown_spec.keys - %i{method timeout}
+			unless bad_keys.empty?
+				raise ArgumentError,
+				      "Invalid key(s) in shutdown specification: #{bad_keys.inspect}"
+			end
+
+			# Restore any missing defaults
+			@shutdown_spec = { method: nil, timeout: 1 }.merge(@shutdown_spec)
+
+			if @shutdown_spec[:method]
+				begin
+					unless @klass.instance_method(@shutdown_spec[:method]).arity == 0
+						raise ArgumentError,
+						      "Shutdown method #{@klass.to_s}##{@shutdown_spec[:method]} must not take any arguments"
+					end
+				rescue NameError
+					raise ArgumentError,
+					      "Shutdown method #{@klass.to_s}##{@shutdown_spec[:method]} is not defined"
+				end
+			end
+
+			unless @shutdown_spec[:timeout].is_a?(Numeric) && !@shutdown_spec[:timeout].negative?
+				raise ArgumentError,
+				      "Invalid shutdown timeout #{@shutdown_spec[:timeout].inspect}: must be non-negative integer"
+			end
+		end
+
+		def validate_access
+			return if @access.nil?
+
+			unless %i{unsafe}.include? @access
+				raise ArgumentError,
+				      "Invalid instance access specification: #{@access.inspect}"
+			end
+		end
+
+		def castcall_enabled?
+			!!@enable_castcall
+		end
+
+		def new_instance
+			# If there is anything that pisses me off about Ruby's varargs handling more
+			# than the fact that *[] is an empty array, and not a zero-length argument
+			# list, I don't know what it is.  Everything else works *so well*, and this...
+			# urgh.
+			if @klass.instance_method(:initialize).arity == 0
+				@klass.new()
+			else
+				@klass.new(*@args)
+			end.tap do |i|
+				if castcall_enabled?
+					i.singleton_class.prepend(Ultravisor::Child::ProcessCastCall)
+					i.instance_variable_set(:@ultravisor_child_castcall_queue, Queue.new)
+
+					r, @castcall_fd_writer = IO.pipe
+					i.instance_variable_set(:@ultravisor_child_castcall_fd, r)
+				end
+			end
+		end
+
+		def current_instance
+			@spawn_m.synchronize do
+				while @instance.nil?
+					@spawn_cv.wait(@spawn_m)
+				end
+
+				return @instance
+			end
+		end
+
+		def blown_policy?
+			cumulative_runtime = 0
+			# This starts at 1 because we only check this during a restart, so
+			# by definition there must have been at least one recent restart
+			recent_restart_count = 1
+
+			@runtime_history.each do |t|
+				cumulative_runtime += t
+
+				if cumulative_runtime < @restart_policy[:period]
+					recent_restart_count += 1
+				end
+			end
+
+			logger.debug(logloc) { "@runtime_history: #{@runtime_history.inspect}, cumulative_runtime: #{cumulative_runtime}, recent_restart_count: #{recent_restart_count}, restart_policy: #{@restart_policy.inspect}" }
+
+			if recent_restart_count > @restart_policy[:max]
+				return true
+			end
+
+			@runtime_history = @runtime_history[0..recent_restart_count]
+
+			false
+		end
+
+		def termination_cleanup(term_queue = nil)
+			unless @spawn_m.owned?
+				#:nocov:
+				raise ThreadSafetyError,
+				      "termination_cleanup must be called while holding the @spawn_m lock"
+				#:nocov:
+			end
+
+			if @start_time
+				@runtime_history.unshift(Time.now.to_f - @start_time.to_f)
+				@start_time = nil
+			end
+
+			term_queue << self if term_queue && !@shutting_down
+
+			if castcall_enabled?
+				cc_q = @instance.instance_variable_get(:@ultravisor_child_castcall_queue)
+				cc_q.close
+				x = 0
+				begin
+					loop do
+						cc_q.pop(true).child_restarted!
+					end
+				rescue ThreadError => ex
+					raise unless ex.message == "queue empty"
+				end
+
+				@instance.instance_variable_get(:@ultravisor_child_castcall_fd).close
+				@instance.instance_variable_set(:@ultravisor_child_castcall_fd, nil)
+				@castcall_fd_writer.close
+				@castcall_fd_writer = nil
+			end
+
+			@instance = nil
+
+			if @thread
+				@thread = nil
+				@spawn_cv.broadcast
+			end
+
+			@spawn_id = nil
+		end
+	end
+end
+
+require_relative "./child/call"
+require_relative "./child/call_receiver"
+require_relative "./child/cast"
+require_relative "./child/cast_receiver"
+require_relative "./child/process_cast_call"