gritz-core 0.2.0 → 0.3.0

data/lib/gritz/supervisor/master.rb

@@ -7,15 +7,19 @@ module Gritz
     # Forks workers, monitors their status pipes, and owns every child until reaped.
     # @api public
     class Master
-      SIGNALS = %w[TERM INT QUIT TTIN TTOU HUP CHLD].freeze
+      SIGNALS = %w[TERM INT QUIT TTIN TTOU HUP CHLD USR1 USR2].freeze
       attr_reader :workers
 
-      def initialize(config, logger: Logger.new($stdout), status_io: nil)
+      def initialize(config, logger: Logger.new($stdout), status_io: nil, owner_channel: nil)
         @config = config
         @logger = logger
         @workers = {}
         @desired = config.workers
-        @reports = StatusChannel.new(status_io) if status_io
+        @owner_channel = owner_channel
+        @reports = owner_channel || (StatusChannel.new(status_io) if status_io)
+        @metrics = Metrics::Aggregator.new
+        @forwarded = []
+        @replacement_queue = []
         @exit_status = 0
       end
 
@@ -25,6 +29,10 @@ module Gritz
 
         require "gritz/native"
         @signals = SignalQueue.new(signals: SIGNALS)
+        unless @owner_channel
+          @admin = AdminServer.new(bind: @config.admin_bind, status: -> { status }, ready: -> { ready? },
+                                   metrics: -> { @metrics.render(workers: @workers.values) }, logger: @logger)
+        end
         @guard = ForkGuard.activate(mode: @config.fork_mode == :clean ? @config.fork_guard : :off, logger: @logger)
         @config.preload! if @config.preload_app?
         Process.warmup if @config.preload_app? && Process.respond_to?(:warmup)
@@ -37,11 +45,25 @@ module Gritz
           read_statuses
           reap_children
           check_timeouts
+          sample_memory
+          check_recycle
+          advance_replacement
           maintain_worker_count unless @shutdown_at
-          @reports&.write(status)
-          break if @shutdown_at && @workers.empty?
+          flush_forwarded
+          @reports&.write(@owner_channel ? status.merge(type: "status") : status) if @forwarded.empty?
+          @admin&.poll
+          if @owner_channel&.closed?
+            @exit_status = 1
+            begin_shutdown(immediate: true)
+          end
+          if @shutdown_at && @workers.empty?
+            break if !@owner_channel || @owner_channel.closed? || (@forwarded.empty? && @owner_channel.flush)
+            break if now >= @shutdown_at + @config.drain_delay + @config.shutdown_timeout
+          end
 
-          IO.select([@signals.io, *@workers.values.filter_map { |handle| handle.channel.io unless handle.channel.closed? }], nil, nil, 0.05)
+          worker_ios = @forwarded.empty? ? @workers.values.filter_map { |handle| handle.channel.io unless handle.channel.closed? } : []
+          owner_ios = !@forwarded.empty? && @owner_channel && !@owner_channel.closed? ? [@owner_channel.io] : nil
+          IO.select([@signals.io, *@admin&.ios.to_a, *worker_ios], owner_ios, nil, 0.05)
         end
         @exit_status
       ensure
@@ -50,7 +72,14 @@ module Gritz
       end
 
       def status
-        { pid: Process.pid, state: @shutdown_at ? "draining" : "running", workers: @workers.values.map(&:to_h) }
+        { pid: Process.pid, state: @shutdown_at ? "draining" : "running", desired: @desired,
+          phased_restart: !@replacement.nil? || !@replacement_queue.empty?, workers: @workers.values.map(&:to_h) }
+      end
+
+      def ready?
+        !@shutdown_at && @workers.values.count { |handle|
+          handle.state == "ready" && !handle.term_at && handle.stats[:healthy] != false
+        } >= @config.min_ready_workers
       end
 
       private
@@ -58,8 +87,9 @@ module Gritz
       def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
       def maintain_worker_count
-        used = @workers.values.map(&:index)
-        (@desired - @workers.size).times do
+        active = @workers.values.reject(&:term_at)
+        used = active.map(&:index)
+        (@desired - active.size).times do
           index = (0...@desired).find { |candidate| !used.include?(candidate) }
           spawn_worker(index)
           used << index
@@ -82,6 +112,7 @@ module Gritz
             # Restore all inherited master traps, including CHLD and resize signals.
             @signals.close
             @reports&.close
+            @admin&.close
             @workers.each_value(&:close)
             Transport::Native.postfork_child if experimental
             exit_status = Worker::Runner.new(index: index, status_io: writer, config: @config, logger: @logger).run
@@ -98,8 +129,11 @@ module Gritz
           end
         end
         writer.close
-        @workers[pid] = WorkerHandle.new(pid: pid, index: index, status_io: reader)
+        handle = WorkerHandle.new(pid: pid, index: index, status_io: reader)
+        handle.recycle_factor = 1.0 + (rand * @config.worker_recycle.fetch(:jitter, 0.0))
+        @workers[pid] = handle
         @logger.info("Worker #{index} spawned pid=#{pid}")
+        handle
       rescue StandardError
         reader&.close unless reader&.closed?
         writer&.close unless writer&.closed?
@@ -110,36 +144,83 @@ module Gritz
 
       def read_statuses
         @workers.each_value do |handle|
-          handle.channel.read.each do |message|
-            handle.update(message, now: now)
-            if message[:state] == "failed" && !@ever_ready
-              @exit_status = 1
-              begin_shutdown(immediate: true)
+          break unless @forwarded.empty?
+
+          consume_status(handle, handle.channel.read)
+          flush_forwarded
+        end
+      end
+
+      def consume_status(handle, rows)
+        rows.each do |message|
+          if message[:type] == "metrics"
+            if @metrics.apply(handle, message) && @owner_channel
+              @forwarded << message.merge(pid: Process.pid, worker_pid: handle.pid, worker_started_at: handle.born_at)
             end
-            @ever_ready = true if message[:state] == "ready"
-          rescue ConfigurationError => e
-            @logger.error("Worker #{handle.pid}: #{e.message}")
-            kill(handle)
+            next
           end
+          handle.update(message, now: now)
+          if message[:state] == "failed" && !@ever_ready
+            @exit_status = 1
+            begin_shutdown
+          end
+          @ever_ready = true if message[:state] == "ready"
+        rescue ConfigurationError => e
+          @logger.error("Worker #{handle.pid}: #{e.message}")
+          kill(handle)
         end
       end
 
+      def flush_forwarded
+        return unless @owner_channel
+
+        @owner_channel.flush
+        @forwarded.shift while !@forwarded.empty? && @owner_channel.write(@forwarded.first)
+      end
+
       def reap_children
+        # Keep a reaped worker's bounded pipe tail queued until owner backpressure clears.
+        return unless @forwarded.empty? || @owner_channel&.closed?
+
         # Only reap owned children: application hooks can start unrelated subprocesses.
         @workers.each_key do |pid|
           result = Process.waitpid2(pid, Process::WNOHANG)
           next unless result
 
-          handle = @workers.delete(pid)
+          handle = @workers.fetch(pid)
+          # A forked application subprocess can retain the writer after the worker exits.
+          # Drain the bytes present at reap time without waiting for that subprocess's EOF.
+          available = 0
+          unless handle.channel.closed?
+            size = [0].pack("i")
+            # FIONREAD reports queued bytes; Ruby 4.0 removed IO#nread.
+            handle.channel.io.ioctl(RUBY_PLATFORM.include?("linux") ? 0x541B : 0x4004667F, size)
+            available = size.unpack1("i")
+          end
+          if available.zero?
+            consume_status(handle, handle.channel.read)
+          end
+          while available.positive?
+            bytes = [available, StatusChannel::MAX_READ_BYTES].min
+            consume_status(handle, handle.channel.read(max_bytes: bytes))
+            available -= bytes
+            break if handle.channel.closed?
+          end
+          @workers.delete(pid)
+          @metrics.forget(handle)
           handle.close
           child_status = result.last
           @logger.info("Worker #{handle.index} exited pid=#{pid} status=#{child_status}")
-          startup_failed = child_status.exited? && !@ever_ready && !@shutdown_at && !handle.term_at
-          shutdown_failed = @shutdown_at && child_status.exited? && !child_status.success?
+          startup_failed = (child_status.exited? || child_status.termsig != Signal.list.fetch("KILL")) && !@ever_ready && !@shutdown_at && !handle.term_at
+          shutdown_failed = @shutdown_at && !child_status.success?
           if handle.state != "killed" && (startup_failed || shutdown_failed)
             @exit_status = 1
             begin_shutdown(immediate: true) if startup_failed
           end
+          if !@shutdown_at && !handle.term_at && @workers.values.count { |worker| !worker.term_at } < @desired
+            record_restart(handle.restart_reason || "worker_exit")
+          end
+          break unless @forwarded.empty?
         rescue Errno::ECHILD
           @workers.delete(pid)&.close
         end
@@ -150,37 +231,136 @@ module Gritz
         when "TERM", "INT" then begin_shutdown
         when "QUIT" then begin_shutdown(immediate: true)
         when "TTIN"
-          if !@shutdown_at && @config.bind.end_with?(":0")
+          if @replacement || !@replacement_queue.empty?
+            @logger.warn("Wait for phased restart before resizing workers")
+          elsif !@shutdown_at && @config.bind.end_with?(":0")
             @logger.warn("Cannot add a reuseport worker with port 0; configure a fixed bind port")
           elsif !@shutdown_at
             @desired += 1
           end
         when "TTOU"
-          if !@shutdown_at && @desired > 1
+          if @replacement || !@replacement_queue.empty?
+            @logger.warn("Wait for phased restart before resizing workers")
+          elsif !@shutdown_at && @desired > 1
             @desired -= 1
-            retire(@workers.values.reject(&:term_at).max_by(&:index))
+            retire(@workers.values.reject(&:term_at).max_by(&:index), delay: 0)
           end
         when "HUP"
           @logger.reopen
           @workers.each_key { |pid| send_signal("HUP", pid) }
           @logger.info("Log reopened")
+        when "USR1"
+          if fixed_port? && !@shutdown_at && !@replacement && @replacement_queue.empty?
+            @replacement_queue = @workers.values.reject(&:term_at).sort_by(&:index).map { |handle| [handle.pid, "phased_restart"] }
+          end
+        when "USR2"
+          if fixed_port? && !@shutdown_at
+            if @owner_channel
+              @forwarded << { type: "reexec", pid: Process.pid } unless @forwarded.any? { |row| row[:type] == "reexec" }
+            else
+              @logger.warn("USR2 requires the gritz launcher")
+            end
+          end
         end
       end
 
       def begin_shutdown(immediate: false)
         @shutdown_at ||= now
+        @replacement_queue.clear
+        @replacement = nil
         if immediate
           @workers.each_value { |handle| kill(handle) }
         else
-          @workers.each_value { |handle| handle.term_at ||= @shutdown_at + @config.drain_delay }
+          @workers.each_value { |handle| retire(handle) unless handle.term_at }
         end
       end
 
-      def retire(handle)
+      def retire(handle, delay: @config.drain_delay)
         return unless handle
 
-        handle.term_at = now
+        handle.term_at = now + delay
         handle.state = "draining"
+        send_signal("USR1", handle.pid)
+      end
+
+      def fixed_port?
+        return true unless @config.bind.end_with?(":0")
+
+        @logger.warn("Worker replacement requires a fixed bind port")
+        false
+      end
+
+      def advance_replacement
+        return if @shutdown_at || !@forwarded.empty?
+
+        if @replacement
+          old = @workers[@replacement[:old]]
+          fresh = @workers[@replacement[:new]]
+          if !fresh || %w[failed stopped killed].include?(fresh.state) ||
+             (!@replacement[:retired] && now - @replacement[:started] > @config.worker_boot_timeout)
+            kill(fresh) if fresh
+            @logger.warn("Replacement failed; keeping previous workers")
+            @replacement = nil
+            @replacement_queue.clear
+            @recycle_retry_at = now + @config.worker_boot_timeout
+            return true
+          end
+          if !@replacement[:retired] && fresh.state == "ready" && fresh.stats[:healthy] != false
+            retire(old) if old
+            reason = @replacement[:reason]
+            record_restart(reason)
+            @replacement[:retired] = true
+          end
+          return if old || !@replacement[:retired]
+
+          @replacement = nil
+        end
+        while (entry = @replacement_queue.shift)
+          old_pid, reason = entry
+          old = @workers[old_pid]
+          next unless old && !old.term_at
+
+          fresh = spawn_worker(old.index)
+          @replacement = { old: old_pid, new: fresh.pid, reason: reason, started: now, retired: false }
+          break
+        end
+      end
+
+      def check_recycle
+        return if @shutdown_at || @replacement || !@replacement_queue.empty? || @config.worker_recycle.empty?
+        return if @recycle_retry_at && now < @recycle_retry_at
+        return if @config.bind.end_with?(":0")
+
+        @workers.each_value do |handle|
+          next unless handle.state == "ready" && !handle.term_at
+
+          @config.worker_recycle.each do |name, limit|
+            value = case name
+                    when :max_requests then handle.stats[:requests_total]
+                    when :max_rss_mb then handle.stats[:rss_bytes]&./(1024.0 * 1024)
+                    when :max_pss_mb then handle.stats[:pss_bytes]&./(1024.0 * 1024)
+                    when :max_lifetime then now - handle.born_at
+                    end
+            next unless value && value >= limit * handle.recycle_factor
+
+            @replacement_queue << [handle.pid, name.to_s]
+            return true
+          end
+        end
+      end
+
+      def sample_memory
+        return unless RUBY_PLATFORM.include?("linux")
+        return if @next_memory_at && now < @next_memory_at
+
+        @next_memory_at = now + @config.status_interval
+        @workers.each_value do |handle|
+          data = File.read("/proc/#{handle.pid}/smaps_rollup")
+          handle.stats[:rss_bytes] = data[/^Rss:\s+(\d+)/, 1].to_i * 1024
+          handle.stats[:pss_bytes] = data[/^Pss:\s+(\d+)/, 1].to_i * 1024
+        rescue Errno::ENOENT, Errno::ESRCH, Errno::EACCES
+          next
+        end
       end
 
       def check_timeouts
@@ -195,11 +375,15 @@ module Gritz
           end
           if handle.kill_at
             kill(handle) if timestamp >= handle.kill_at
-          elsif !@shutdown_at
+          elsif !@shutdown_at && !handle.term_at
+            # An unconsumed heartbeat is not evidence of a stuck worker.
+            next if handle.state != "booting" && !@forwarded.empty?
+
             elapsed = timestamp - (handle.state == "booting" ? handle.born_at : handle.last_seen)
             timeout = handle.state == "booting" ? @config.worker_boot_timeout : @config.worker_timeout
             if elapsed > timeout
               @logger.error("Worker #{handle.pid} #{handle.state} timeout after #{elapsed.round(2)}s")
+              handle.restart_reason = handle.state == "booting" ? "worker_boot_timeout" : "worker_timeout"
               kill(handle)
             end
           end
@@ -211,6 +395,11 @@ module Gritz
         send_signal("KILL", handle.pid)
       end
 
+      def record_restart(reason)
+        @metrics.record_restart(reason: reason)
+        @forwarded << { type: "restart", pid: Process.pid, reason: reason } if @owner_channel
+      end
+
       def send_signal(signal, pid)
         Process.kill(signal, pid)
       rescue Errno::ESRCH
@@ -228,6 +417,8 @@ module Gritz
         end
         @workers.clear
         @signals&.close
+        @admin&.close
+        flush_forwarded
         @reports&.close
       end
     end

data/lib/gritz/supervisor/status_channel.rb

@@ -28,7 +28,8 @@ module Gritz
         return false if line.bytesize > MAX_LINE_BYTES
 
         @pending = line
-        flush_pending.zero?
+        flush_pending
+        true
       rescue JSON::GeneratorError, JSON::NestingError
         false
       rescue IOError, SystemCallError
@@ -36,11 +37,22 @@ module Gritz
         false
       end
 
-      def read
+      # True means the accepted record is fully delivered; false means retry later.
+      def flush
+        return false if closed?
+
+        flush_pending.zero?
+      rescue IOError, SystemCallError
+        close
+        false
+      end
+
+      def read(max_bytes: MAX_READ_BYTES)
         rows = []
         return rows if closed?
+        raise ArgumentError, "Invalid status read budget" unless max_bytes.is_a?(Integer) && max_bytes.between?(1, MAX_READ_BYTES)
 
-        remaining = MAX_READ_BYTES
+        remaining = max_bytes
         while remaining.positive?
           chunk = @io.read_nonblock([4096, remaining].min, exception: false)
           break if chunk == :wait_readable

data/lib/gritz/supervisor/worker_handle.rb

@@ -7,7 +7,7 @@ module Gritz
     class WorkerHandle
       STATES = %w[booting ready draining failed stopped killed].freeze
       attr_reader :pid, :index, :channel, :born_at, :last_seen, :stats
-      attr_accessor :state, :term_at, :kill_at
+      attr_accessor :state, :term_at, :kill_at, :recycle_factor, :restart_reason
 
       def initialize(pid:, index:, status_io:, now: Process.clock_gettime(Process::CLOCK_MONOTONIC))
         @pid = pid
@@ -16,18 +16,19 @@ module Gritz
         @born_at = @last_seen = now
         @state = "booting"
         @stats = {}
+        @recycle_factor = 1.0
       end
 
       def update(message, now:)
         raise ConfigurationError, "Invalid worker state #{message[:state].inspect}" unless STATES.include?(message[:state])
 
         @last_seen = now
-        @stats = message.except(:pid, :index, :state)
+        @stats = @stats.slice(:rss_bytes, :pss_bytes).merge(message.except(:pid, :index, :state, :type, :metrics))
         # A late heartbeat must not undo the parent's decision to retire this worker.
         @state = message[:state] unless %w[draining killed].include?(@state)
       end
 
-      def to_h = @stats.merge(pid: @pid, index: @index, state: @state)
+      def to_h = @stats.merge(pid: @pid, index: @index, state: @state, retiring: !@term_at.nil?, worker_started_at: @born_at)
       def close = @channel.close
     end
   end

data/lib/gritz/testing/cluster.rb

@@ -35,12 +35,13 @@ module Gritz
       def start
         raise ArgumentError, "cluster is already started" if @pid
 
+        Supervisor::Launcher.enable_subreaper!
         @reader, writer = IO.pipe
         @channel = Supervisor::StatusChannel.new(@reader)
         @log = Tempfile.new(["gritz-cluster", ".log"])
         command = @command || [
           RbConfig.ruby, "-I", $LOAD_PATH.join(File::PATH_SEPARATOR), "-rgritz/core", "-e",
-          "exit Gritz::CLI.new(status_io: IO.for_fd(3)).run(ARGV)", "--", "start", "-C", @config_path
+          "exit Gritz::CLI.new(status_io: IO.for_fd(3), launch: true).run(ARGV)", "--", "start", "-C", @config_path
         ]
         @pid = Process.spawn(@env, *command, 3 => writer, out: @log, err: @log, pgroup: true)
         self
@@ -61,6 +62,8 @@ module Gritz
 
       def workers = status.fetch(:workers, [])
 
+      def master_pid = status[:pid] || @pid
+
       def logs
         return @logs || "" unless @log && !@log.closed?
 
@@ -83,12 +86,13 @@ module Gritz
         deadline = monotonic + timeout
         loop do
           snapshot = status
+          active_workers = snapshot.fetch(:workers, []).reject { |worker| worker[:retiring] }
           ready = if state == "ready"
-                    snapshot[:state] == "running" && snapshot[:workers]&.any? && snapshot[:workers].all? { |worker| worker[:state] == "ready" }
+                    snapshot[:state] == "running" && active_workers.any? && active_workers.all? { |worker| worker[:state] == "ready" }
                   else
                     state.nil? || snapshot[:state] == state
                   end
-          count = workers.nil? || snapshot.fetch(:workers, []).size == workers
+          count = workers.nil? || active_workers.size == workers
           return self if ready && count && (!block_given? || yield(snapshot))
 
           if exited?
@@ -120,14 +124,16 @@ module Gritz
         wait(timeout:)
         self
       rescue Timeout::Error
+        signal("QUIT") if status[:owner_pid] && !exited?
         begin
-          Process.kill("KILL", -@pid)
-        rescue Errno::ESRCH
-          # The process group may have exited between the timeout and kill.
+          wait(timeout: 0.5)
+        rescue Timeout::Error
+          owned_groups.each { |pid| kill_group(pid) }
+          wait(timeout: 5)
         end
-        wait(timeout: 5)
         self
       ensure
+        cleanup_groups if @pid && @exit_status
         @channel&.close
         @logs = logs
         @log&.close!
@@ -137,6 +143,36 @@ module Gritz
 
       def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
+      def owned_groups
+        [@pid, *status.fetch(:masters, []).map { |master| master[:pid] }].compact.uniq
+      end
+
+      def kill_group(pid)
+        Process.kill("KILL", -pid)
+      rescue Errno::ESRCH
+        nil
+      rescue Errno::EPERM
+        # Darwin may report EPERM for an already-disappeared process group.
+        begin
+          Process.kill(0, pid)
+        rescue Errno::ESRCH
+          return
+        end
+        raise
+      end
+
+      def cleanup_groups
+        groups = owned_groups.reject { |pid| pid == @pid }
+        groups.each { |pid| kill_group(pid) }
+        # Terminate every group before waiting for any adopted children.
+        groups.each do |pid| # rubocop:disable Style/CombinableLoops
+          loop { Process.waitpid(-pid) }
+        rescue Errno::ECHILD
+          nil
+        end
+        @status = @status.merge(masters: [])
+      end
+
       def exited?
         return true if @exit_status
         return false unless @pid