gritz-core 0.1.0 → 0.3.0

data/lib/gritz/metrics/recorder.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Gritz
+  module Metrics
+    # Per-worker RPC deltas; only one bounded detached batch is needed by the sender.
+    # @api private
+    class Recorder
+      DURATION_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, Float::INFINITY].freeze
+      MESSAGE_BUCKETS = [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 1024, Float::INFINITY].freeze
+      SCALARS = %i[count duration_sum request_sum response_sum].freeze
+      BUCKET_FIELDS = %i[duration_buckets request_buckets response_buckets].freeze
+
+      def initialize
+        @lock = Mutex.new
+        @rpc = {}
+        @last_rejected = @rejected_delta = 0
+      end
+
+      def self.empty_row(service:, method:, code:)
+        { service: service.dup.freeze, method: method.dup.freeze, code: code, count: 0, duration_sum: 0.0, request_sum: 0, response_sum: 0,
+          duration_buckets: Array.new(DURATION_BUCKETS.size, 0), request_buckets: Array.new(MESSAGE_BUCKETS.size, 0),
+          response_buckets: Array.new(MESSAGE_BUCKETS.size, 0) }
+      end
+
+      def record_rpc(service:, method:, code:, duration:, requests:, responses:)
+        unless [service, method].all? { |label| label.is_a?(String) && !label.empty? } && code.is_a?(Integer) && code.between?(0, 16) &&
+               duration.is_a?(Numeric) && duration.real? && duration.to_f.finite? && duration >= 0 &&
+               [requests, responses].all? { |value| value.is_a?(Integer) && value >= 0 }
+          raise ArgumentError, "Invalid RPC metric observation"
+        end
+
+        @lock.synchronize do
+          row = @rpc[[service, method, code]]
+          unless row
+            row = self.class.empty_row(service: service, method: method, code: code)
+            @rpc[[row[:service], row[:method], code].freeze] = row
+          end
+          row[:count] += 1
+          row[:duration_sum] += duration.to_f
+          row[:request_sum] += requests
+          row[:response_sum] += responses
+          row[:duration_buckets][DURATION_BUCKETS.bsearch_index { |upper| duration <= upper }] += 1
+          row[:request_buckets][MESSAGE_BUCKETS.bsearch_index { |upper| requests <= upper }] += 1
+          row[:response_buckets][MESSAGE_BUCKETS.bsearch_index { |upper| responses <= upper }] += 1
+        end
+      end
+
+      def observe_rejected(total)
+        @lock.synchronize do
+          raise ArgumentError, "Rejected counter must be a nondecreasing integer" unless total.is_a?(Integer) && total >= @last_rejected
+
+          @rejected_delta += total - @last_rejected
+          @last_rejected = total
+        end
+      end
+
+      # The caller retains this detached delta until its status channel accepts the row.
+      def take_delta(max_bytes: Supervisor::StatusChannel::MAX_LINE_BYTES - 4096)
+        raise ArgumentError, "Metric packet budget must be positive" unless max_bytes.is_a?(Integer) && max_bytes.positive?
+
+        @lock.synchronize do
+          return nil if @rpc.empty? && @rejected_delta.zero?
+
+          delta = { rpc: [], rejected: @rejected_delta }
+          used_bytes = JSON.generate(delta).bytesize
+          raise ArgumentError, "Metric packet budget is too small" if used_bytes > max_bytes
+
+          selected = []
+          @rpc.each do |key, row|
+            row_bytes = JSON.generate(row).bytesize + (selected.empty? ? 0 : 1)
+            if used_bytes + row_bytes > max_bytes
+              raise ArgumentError, "Metric row exceeds the packet budget" if selected.empty?
+
+              break
+            end
+            delta[:rpc] << row
+            selected << key
+            used_bytes += row_bytes
+          end
+          selected.each { |key| @rpc.delete(key) }
+          @rejected_delta = 0
+          delta
+        end
+      end
+    end
+  end
+end

data/lib/gritz/middleware/exception_mapper.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "securerandom"
-require "json"
 
 module Gritz
   module Middleware
@@ -32,8 +31,7 @@ module Gritz
         end
 
         error_id = SecureRandom.uuid
-        context.logger.error(JSON.generate(error_id:, request_id: context.request_id, error: e.class.name,
-                                           message: e.message, backtrace: e.backtrace))
+        context.store[:gritz_error] = { error_id: error_id, error: e.class.name, message: e.message, backtrace: e.backtrace }
         raise Errors::Internal.new(@expose_errors ? e.message : "internal error (#{error_id})", metadata: { "error-id" => error_id })
       end
     end

data/lib/gritz/middleware/logging.rb

@@ -23,9 +23,33 @@ module Gritz
         raise
       ensure
         duration = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000
-        context.logger.info(JSON.generate(request_id: context.request_id, service: context.method.service,
-                                          method: context.method.name, code:, duration_ms: duration.round(3),
-                                          peer: context.peer, pid: Process.pid))
+        fields = { request_id: context.request_id, service: context.method.service,
+                   method: context.method.name, code: code, duration_ms: duration.round(3),
+                   peer: context.peer, worker: context.worker, pid: Process.pid, bytes_in: context.bytes_in, bytes_out: context.bytes_out }
+        fields.merge!(context.store.fetch(:gritz_error, {}))
+        fields = prepare(fields, context.log_redact)
+        message = if context.log_format == :logfmt
+                    fields.map do |key, value|
+                      value = value.to_s if value.is_a?(Symbol)
+                      value = JSON.generate(value) if value.is_a?(Hash) || value.is_a?(Array)
+                      "#{key}=#{JSON.generate(value)}"
+                    end.join(" ")
+                  else
+                    JSON.generate(fields)
+                  end
+        context.logger.info(message)
+      end
+
+      private
+
+      def prepare(value, redacted)
+        case value
+        when Hash
+          value.to_h { |key, item| [key, redacted.include?(key.to_s) ? "[FILTERED]" : prepare(item, redacted)] }
+        when Array then value.map { |item| prepare(item, redacted) }
+        when String then value.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+        else value
+        end
       end
     end
   end

data/lib/gritz/middleware/metrics.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gritz
+  module Middleware
+    # One completion observation wraps the complete unary or streaming action.
+    # @api public
+    class Metrics
+      def initialize(app)
+        @app = app
+      end
+
+      def call(context)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        code = 0
+        @app.call(context)
+      rescue Gritz::Error => e
+        code = e.grpc_code
+        raise
+      rescue StandardError
+        code = 13
+        raise
+      ensure
+        context.metrics&.record_rpc(service: context.method.service, method: context.method.name, code: code,
+                                    duration: Process.clock_gettime(Process::CLOCK_MONOTONIC) - started,
+                                    requests: context.requests_count, responses: context.responses_count)
+      end
+    end
+  end
+end

data/lib/gritz/middleware/stack.rb

@@ -18,7 +18,7 @@ module Gritz
       end
 
       def self.default
-        new.use(RequestId).use(Context).use(Logging).use(ExceptionMapper)
+        new.use(RequestId).use(Context).use(Metrics).use(Logging).use(ExceptionMapper)
       end
 
       def use(middleware, **options, &block)

data/lib/gritz/supervisor/admin_server.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require "socket"
+require "json"
+
+module Gritz
+  module Supervisor
+    # Small, bounded HTTP probe listener polled by its owning process.
+    # @api private
+    class AdminServer
+      MAX_CLIENTS = 32
+      MAX_HEADER_BYTES = 8192
+      CLIENT_TIMEOUT = 2.0
+      attr_reader :io, :address
+
+      def initialize(bind:, status:, ready:, metrics:, logger:)
+        host, port = bind.match(/\A(\[[^\]]+\]|[^:]+):(\d+)\z/).captures
+        host = host.delete_prefix("[").delete_suffix("]")
+        @io = TCPServer.new(host, Integer(port))
+        @address = "#{host.include?(':') ? "[#{host}]" : host}:#{@io.addr[1]}"
+        @status = status
+        @ready = ready
+        @metrics = metrics
+        @logger = logger
+        @clients = {}
+      end
+
+      def ios = [@io, *@clients.keys]
+
+      def poll
+        4.times do
+          socket = @io.accept_nonblock(exception: false)
+          break if socket == :wait_readable
+
+          if @clients.size >= MAX_CLIENTS
+            socket.close
+          else
+            @clients[socket] = { input: +"".b, output: nil, deadline: monotonic + CLIENT_TIMEOUT }
+          end
+        rescue Errno::ECONNABORTED
+          next
+        end
+        @clients.each do |socket, client|
+          if monotonic >= client[:deadline]
+            drop(socket)
+            next
+          end
+          read_request(socket, client) unless client[:output]
+          write_response(socket, client) if client[:output]
+        rescue IOError, SystemCallError
+          drop(socket)
+        end
+      end
+
+      def close
+        @clients.each_key(&:close)
+        @clients.clear
+        @io.close unless @io.closed?
+      end
+
+      private
+
+      def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      def drop(socket)
+        @clients.delete(socket)
+        socket.close unless socket.closed?
+      end
+
+      def read_request(socket, client)
+        chunk = socket.read_nonblock(4096, exception: false)
+        return if chunk == :wait_readable
+        return drop(socket) if chunk.nil?
+
+        client[:input] << chunk
+        if client[:input].bytesize > MAX_HEADER_BYTES
+          respond(client, 431, "Request headers too large\n")
+        elsif client[:input].include?("\r\n\r\n")
+          method, path, version = client[:input].lines.first.split
+          if !%w[HTTP/1.0 HTTP/1.1].include?(version)
+            respond(client, 400, "Invalid request\n")
+          elsif method != "GET"
+            respond(client, 405, "GET required\n")
+          else
+            route(client, path)
+          end
+        end
+      end
+
+      def route(client, path)
+        case path
+        when "/livez" then respond(client, 200, "ok\n")
+        when "/readyz"
+          ready = @ready.call
+          respond(client, ready ? 200 : 503, ready ? "ready\n" : "not ready\n")
+        when "/status" then respond(client, 200, "#{JSON.generate(@status.call)}\n", "application/json")
+        when "/metrics" then respond(client, 200, @metrics.call, "text/plain; version=0.0.4")
+        else respond(client, 404, "Not found\n")
+        end
+      rescue StandardError => e
+        @logger.error("Admin #{path}: #{e.message}")
+        respond(client, 500, "Internal error\n")
+      end
+
+      def respond(client, code, body, content_type = "text/plain")
+        reasons = { 200 => "OK", 400 => "Bad Request", 404 => "Not Found", 405 => "Method Not Allowed",
+                    431 => "Request Header Fields Too Large", 500 => "Internal Server Error", 503 => "Service Unavailable" }
+        client[:output] =
+          "HTTP/1.1 #{code} #{reasons.fetch(code)}\r\nContent-Type: #{content_type}\r\nContent-Length: #{body.bytesize}\r\nConnection: close\r\n\r\n#{body}"
+      end
+
+      def write_response(socket, client)
+        written = socket.write_nonblock(client[:output], exception: false)
+        return if written == :wait_writable
+
+        client[:output] = client[:output].byteslice(written..)
+        drop(socket) if client[:output].empty?
+      end
+    end
+  end
+end

data/lib/gritz/supervisor/launcher.rb

@@ -0,0 +1,381 @@
+# frozen_string_literal: true
+
+require "socket"
+require "fileutils"
+require "tempfile"
+
+module Gritz
+  module Supervisor
+    # Keeps process ownership and probes stable while fresh master interpreters replace each other.
+    # @api private
+    class Launcher
+      Generation = Struct.new(:pid, :channel, :token, :role, :metadata, :snapshot, :deadline,
+                              :kill_at, :kill_sent, :exit_status, :group_reaped, :identities, keyword_init: true)
+      SIGNALS = %w[TERM INT QUIT USR1 USR2 TTIN TTOU HUP CHLD].freeze
+
+      def self.enable_subreaper!
+        return unless RUBY_PLATFORM.include?("linux")
+
+        require "fiddle"
+        function = Fiddle::Function.new(Fiddle.dlopen(nil)["prctl"],
+                                        [Fiddle::TYPE_INT, *Array.new(4, Fiddle::TYPE_LONG)], Fiddle::TYPE_INT)
+        raise "Cannot enable child process reaping" unless function.call(36, 1, 0, 0, 0).zero?
+      end
+
+      def initialize(command:, logger:, env: ENV, stdout: $stdout, stderr: $stderr, status_io: nil, startup_timeout: 60)
+        @command = command
+        @logger = logger
+        @env = env
+        @stdout = stdout
+        @stderr = stderr
+        @observer = StatusChannel.new(status_io) if status_io
+        @startup_timeout = startup_timeout
+        @generations = {}
+        @serial = 0
+        @metrics = Metrics::Aggregator.new
+        @exit_code = 0
+      end
+
+      def run
+        self.class.enable_subreaper!
+        @signals = SignalQueue.new(signals: SIGNALS)
+        spawn_generation
+        loop do
+          @signals.drain.each { |name| handle_signal(name) }
+          # Reading a reexec request can add a generation, so iterate a stable snapshot.
+          @generations.values.each do |generation| # rubocop:disable Style/HashEachMethods
+            read_generation(generation)
+            reap_generation(generation)
+          end
+          update_lifecycle
+          @admin&.poll
+          @observer&.write(status)
+          @observer&.flush
+          break if @stopping && @generations.empty?
+
+          ios = [@signals.io, *@generations.values.reject { |generation| generation.channel.closed? }.map { |generation| generation.channel.io },
+                 *@admin&.ios].compact
+          IO.select(ios, nil, nil, 0.02)
+        end
+        @exit_code
+      rescue StandardError => e
+        @logger.error("Launcher failed: #{e.message}")
+        1
+      ensure
+        cleanup
+      end
+
+      def status
+        active = @active&.snapshot || {}
+        pools = @generations.values.select { |generation| %i[active retiring].include?(generation.role) || (!@active && generation == @candidate) }
+        workers = pools.flat_map do |generation|
+          generation.snapshot&.fetch(:workers, [])&.map do |worker|
+            worker.merge(master_pid: generation.pid, retiring: generation.role == :retiring || worker[:retiring] == true)
+          end || []
+        end
+        state = if @stopping
+                  "draining"
+                elsif @active
+                  "running"
+                else
+                  "starting"
+                end
+        active.merge(owner_pid: Process.pid, pid: (@active || @candidate)&.pid, state: state,
+                     workers: workers, ready_workers: ready_workers(@active), admin_address: @admin&.address,
+                     masters: @generations.values.map { |generation| { pid: generation.pid, state: generation.role.to_s } }, reexec: @reexec)
+      end
+
+      private
+
+      def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      def spawn_generation
+        parent, child = UNIXSocket.pair
+        pid = Process.spawn(@env.to_h.merge("GRITZ_INTERNAL_OWNER_FD" => "3"), *@command,
+                            3 => child, out: @stdout, err: @stderr, pgroup: true)
+        @serial += 1
+        @candidate = Generation.new(pid: pid, channel: StatusChannel.new(parent), token: @serial, role: :candidate,
+                                    deadline: monotonic + @startup_timeout, identities: {})
+        @generations[pid] = @candidate
+        @reexec = { state: "starting", pid: pid } if @active
+      rescue StandardError
+        parent&.close
+        raise
+      ensure
+        child&.close
+      end
+
+      def read_generation(generation)
+        generation.channel.read.each do |row|
+          next unless row[:pid] == generation.pid
+
+          case row[:type]
+          when "configured" then configure_generation(generation, row)
+          when "status"
+            generation.snapshot = row.except(:type)
+            live_pids = row.fetch(:workers, []).map { |worker| worker[:pid] }
+            generation.identities.delete_if do |pid, key|
+              next false if live_pids.include?(pid)
+
+              @metrics.forget(key)
+              true
+            end
+          when "reexec" then request_reexec if generation == @active
+          when "restart" then @metrics.record_restart(reason: row[:reason])
+          when "metrics"
+            previous = generation.identities[row[:worker_pid]]
+            key = [generation.token, row[:worker_pid], row[:worker_started_at]].freeze
+            @metrics.forget(previous) if previous && previous != key
+            generation.identities[row[:worker_pid]] = key
+            @metrics.apply(key, row.slice(:seq, :delta))
+          end
+        end
+      rescue ConfigurationError, ArgumentError, SystemCallError => e
+        fail_candidate(e.message) if generation == @candidate
+        @logger.error("Master #{generation.pid}: #{e.message}")
+      end
+
+      def configure_generation(generation, metadata)
+        return if generation.metadata || @stopping
+
+        %i[admin_bind bind pid_file].each do |field|
+          raise ConfigurationError, "Missing master #{field}" unless metadata[field].is_a?(String)
+        end
+        %i[workers min_ready_workers].each do |field|
+          raise ConfigurationError, "Invalid master #{field}" unless metadata[field].is_a?(Integer) && metadata[field] >= 0
+        end
+        %i[reexec_timeout drain_delay shutdown_timeout].each do |field|
+          value = metadata[field]
+          raise ConfigurationError, "Invalid master #{field}" unless value.is_a?(Numeric) && value.finite? && value >= 0
+        end
+        if @metadata
+          %i[admin_bind bind pid_file].each do |field|
+            raise ConfigurationError, "USR2 cannot change #{field}" unless metadata[field] == @metadata[field]
+          end
+        else
+          @metadata = metadata
+          @admin = AdminServer.new(bind: metadata[:admin_bind], status: -> { status }, ready: -> { ready? },
+                                   metrics: -> { @metrics.render(workers: status[:workers]) }, logger: @logger)
+          lock_pid_file(metadata[:pid_file]) unless metadata[:pid_file].empty?
+        end
+        generation.metadata = metadata
+        generation.deadline = [generation.deadline, monotonic + metadata[:reexec_timeout]].min
+      end
+
+      def ready_workers(generation)
+        return 0 unless generation&.snapshot&.dig(:state) == "running"
+
+        generation.snapshot.fetch(:workers, []).count do |worker|
+          worker[:state] == "ready" && worker[:healthy] != false && worker[:retiring] != true
+        end
+      end
+
+      def booted_workers(generation)
+        return 0 unless generation&.snapshot&.dig(:state) == "running"
+
+        generation.snapshot.fetch(:workers, []).count { |worker| worker[:state] == "ready" && worker[:retiring] != true }
+      end
+
+      def ready?
+        !@stopping && @active && ready_workers(@active) >= @active.metadata[:min_ready_workers]
+      end
+
+      def request_reexec
+        return if @stopping
+
+        if @candidate || @generations.values.any? { |generation| generation.role == :retiring }
+          @pending_reexec = true
+          return
+        end
+
+        if @active.metadata[:bind].end_with?(":0")
+          @reexec = { state: "failed", error: "USR2 requires a fixed RPC port" }
+          @logger.warn(@reexec[:error])
+          return
+        end
+        spawn_generation
+      rescue StandardError => e
+        @reexec = { state: "failed", error: e.message }
+        @logger.error("USR2 failed: #{e.message}")
+      end
+
+      def update_lifecycle
+        if @candidate&.role == :candidate
+          if @candidate.exit_status
+            fail_candidate("replacement master exited before readiness", force: false)
+          elsif @candidate.channel.closed?
+            fail_candidate("replacement master closed its control channel before readiness", force: false)
+          elsif monotonic >= @candidate.deadline
+            fail_candidate("replacement master readiness timed out")
+          elsif @candidate.metadata && (@active ? ready_workers(@candidate) : booted_workers(@candidate)) >= [@candidate.metadata[:workers], 1].max
+            promote_candidate
+          end
+        end
+        if @active&.exit_status && !@stopping
+          begin_shutdown(code: @active.exit_status.success? ? 0 : 1)
+        elsif @active&.snapshot&.dig(:state) == "draining" && !@stopping
+          begin_shutdown
+        end
+        if @active&.exit_status&.exited? && !@active.exit_status.success?
+          @exit_code = 1
+        end
+        @generations.each_value do |generation|
+          kill_group(generation) if generation.kill_at && monotonic >= generation.kill_at
+          next unless generation.group_reaped && generation.channel.closed?
+
+          generation.identities.each_value { |key| @metrics.forget(key) }
+          @generations.delete(generation.pid)
+          @candidate = nil if @candidate == generation
+        end
+        if @pending_reexec && !@stopping && !@candidate && @generations.values.none? { |generation| generation.role == :retiring }
+          @pending_reexec = false
+          request_reexec
+        end
+      end
+
+      def promote_candidate
+        replacement = @candidate
+        write_pid(replacement.pid)
+        if @active
+          @active.role = :retiring
+          terminate(@active)
+          @metrics.record_restart(reason: "hot_reexec")
+          @reexec = { state: "complete", pid: replacement.pid }
+        end
+        replacement.role = :active
+        @active = replacement
+        @candidate = nil
+      rescue StandardError => e
+        fail_candidate(e.message)
+      end
+
+      def fail_candidate(message, force: true)
+        return unless @candidate
+
+        @candidate.role = :failed
+        if force
+          kill_group(@candidate)
+        else
+          @candidate.kill_at = monotonic + 1
+        end
+        @reexec = { state: "failed", error: message }
+        @logger.error("USR2 failed: #{message}")
+        unless @active
+          @exit_code = 1
+          @stopping = true
+        end
+      end
+
+      def reap_generation(generation)
+        reap_exited_group(generation)
+        kill_group(generation) if generation.exit_status && !generation.group_reaped
+      end
+
+      def reap_exited_group(generation)
+        loop do
+          result = Process.waitpid2(-generation.pid, Process::WNOHANG)
+          break unless result
+
+          generation.exit_status = result.last if result.first == generation.pid
+        end
+      rescue Errno::ECHILD
+        generation.group_reaped = true
+      end
+
+      def handle_signal(name)
+        case name
+        when "TERM", "INT" then begin_shutdown
+        when "QUIT"
+          @stopping = true
+          @candidate.role = :stopping if @candidate
+          @generations.each_value { |generation| kill_group(generation) }
+        when "USR2" then request_reexec if @active
+        when "HUP"
+          @logger.reopen if @logger.respond_to?(:reopen)
+          @generations.each_value { |generation| signal_generation(generation, name) if %i[active retiring].include?(generation.role) }
+        when "USR1"
+          if @active && !@stopping
+            if @active.metadata[:workers].zero?
+              @logger.warn("USR1 requires worker processes; use USR2 to reload a single-process server")
+            else
+              signal_generation(@active, name)
+            end
+          end
+        when "TTIN", "TTOU" then signal_generation(@active, name) if @active && !@stopping
+        end
+      end
+
+      def begin_shutdown(code: 0)
+        @exit_code = code if code != 0
+        return if @stopping
+
+        @stopping = true
+        @generations.each_value do |generation|
+          generation.role = :stopping if generation.role == :candidate
+          terminate(generation)
+        end
+      end
+
+      def terminate(generation)
+        signal_generation(generation, "TERM")
+        metadata = generation.metadata || {}
+        generation.kill_at = monotonic + metadata.fetch(:drain_delay, 0) + metadata.fetch(:shutdown_timeout, 1) + 1
+      end
+
+      def signal_generation(generation, name)
+        Process.kill(name, generation.pid) unless generation.exit_status
+      rescue Errno::ESRCH
+        nil
+      end
+
+      def kill_group(generation)
+        reap_exited_group(generation)
+        return if generation.group_reaped || generation.kill_sent
+
+        Process.kill("KILL", -generation.pid)
+        generation.kill_sent = true
+        generation.kill_at = nil
+      rescue Errno::ESRCH
+        generation.kill_sent = true
+      end
+
+      def lock_pid_file(path)
+        @pid_path = File.expand_path(path)
+        FileUtils.mkdir_p(File.dirname(@pid_path))
+        @pid_lock = File.open("#{@pid_path}.lock", File::RDWR | File::CREAT, 0o644)
+        raise ConfigurationError, "PID file is already owned: #{@pid_path}" unless @pid_lock.flock(File::LOCK_EX | File::LOCK_NB)
+      end
+
+      def write_pid(pid)
+        return unless @pid_path
+
+        Tempfile.create([".gritz-pid", ".tmp"], File.dirname(@pid_path)) do |file|
+          file.write("#{pid}\n")
+          file.flush
+          File.rename(file.path, @pid_path)
+        end
+        @written_pid = pid
+      end
+
+      def cleanup
+        @generations.each_value do |generation|
+          kill_group(generation)
+          generation.channel.close
+          begin
+            loop { Process.waitpid(-generation.pid) }
+          rescue Errno::ECHILD
+            nil
+          end
+        end
+        @generations.clear
+        @admin&.close
+        @observer&.close
+        @signals&.close
+        if @written_pid && File.file?(@pid_path) && File.read(@pid_path).strip == @written_pid.to_s
+          File.unlink(@pid_path)
+        end
+        @pid_lock&.close
+      end
+    end
+  end
+end