gritz-core 0.1.0 → 0.3.0

data/lib/gritz/testing/cluster.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require "rbconfig"
+require "tempfile"
+require "timeout"
+require "io/wait"
+
+module Gritz
+  module Testing
+    # Starts a supervisor in a fresh interpreter, avoiding gRPC state inherited from a test process.
+    # @api public
+    class Cluster
+      MAX_LOG_BYTES = 64 * 1024
+
+      attr_reader :pid
+
+      def self.start(**)
+        cluster = new(**).start
+        return cluster unless block_given?
+
+        begin
+          yield cluster
+        ensure
+          cluster.stop
+        end
+      end
+
+      def initialize(config_path:, env: {}, command: nil)
+        @config_path = File.expand_path(config_path)
+        @env = env
+        @command = command
+        @status = { state: "starting", workers: [] }
+      end
+
+      def start
+        raise ArgumentError, "cluster is already started" if @pid
+
+        Supervisor::Launcher.enable_subreaper!
+        @reader, writer = IO.pipe
+        @channel = Supervisor::StatusChannel.new(@reader)
+        @log = Tempfile.new(["gritz-cluster", ".log"])
+        command = @command || [
+          RbConfig.ruby, "-I", $LOAD_PATH.join(File::PATH_SEPARATOR), "-rgritz/core", "-e",
+          "exit Gritz::CLI.new(status_io: IO.for_fd(3), launch: true).run(ARGV)", "--", "start", "-C", @config_path
+        ]
+        @pid = Process.spawn(@env, *command, 3 => writer, out: @log, err: @log, pgroup: true)
+        self
+      rescue StandardError
+        unless @pid
+          @channel&.close
+          @log&.close!
+        end
+        raise
+      ensure
+        writer&.close
+      end
+
+      def status
+        @channel&.read&.each { |row| @status = row }
+        @status
+      end
+
+      def workers = status.fetch(:workers, [])
+
+      def master_pid = status[:pid] || @pid
+
+      def logs
+        return @logs || "" unless @log && !@log.closed?
+
+        @log.flush
+        File.open(@log.path) do |file|
+          file.seek([file.size - MAX_LOG_BYTES, 0].max)
+          file.read
+        end
+      end
+
+      def signal(name, pid: @pid)
+        raise ArgumentError, "cluster is not started" unless pid
+
+        Process.kill(name, pid)
+        self
+      end
+
+      # A predicate can inspect snapshots without fixed sleeps or dependence on log wording.
+      def wait_until(state: "ready", workers: nil, timeout: 10)
+        deadline = monotonic + timeout
+        loop do
+          snapshot = status
+          active_workers = snapshot.fetch(:workers, []).reject { |worker| worker[:retiring] }
+          ready = if state == "ready"
+                    snapshot[:state] == "running" && active_workers.any? && active_workers.all? { |worker| worker[:state] == "ready" }
+                  else
+                    state.nil? || snapshot[:state] == state
+                  end
+          count = workers.nil? || active_workers.size == workers
+          return self if ready && count && (!block_given? || yield(snapshot))
+
+          if exited?
+            raise "cluster exited #{@exit_status.inspect} before reaching #{state.inspect}\n#{logs}"
+          end
+          raise Timeout::Error, "cluster did not reach #{state.inspect}: #{snapshot.inspect}\n#{logs}" if monotonic >= deadline
+
+          pause = (deadline - monotonic).clamp(0, 0.05)
+          @reader.closed? ? sleep(pause) : @reader.wait_readable(pause)
+        end
+      end
+
+      def wait(timeout: 10)
+        raise ArgumentError, "cluster is not started" unless @pid
+
+        deadline = monotonic + timeout
+        until exited?
+          raise Timeout::Error, "cluster did not exit\n#{logs}" if monotonic >= deadline
+
+          sleep((deadline - monotonic).clamp(0, 0.01))
+        end
+        @exit_status
+      end
+
+      def stop(timeout: 10)
+        return self unless @pid
+
+        signal("TERM") unless exited?
+        wait(timeout:)
+        self
+      rescue Timeout::Error
+        signal("QUIT") if status[:owner_pid] && !exited?
+        begin
+          wait(timeout: 0.5)
+        rescue Timeout::Error
+          owned_groups.each { |pid| kill_group(pid) }
+          wait(timeout: 5)
+        end
+        self
+      ensure
+        cleanup_groups if @pid && @exit_status
+        @channel&.close
+        @logs = logs
+        @log&.close!
+      end
+
+      private
+
+      def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      def owned_groups
+        [@pid, *status.fetch(:masters, []).map { |master| master[:pid] }].compact.uniq
+      end
+
+      def kill_group(pid)
+        Process.kill("KILL", -pid)
+      rescue Errno::ESRCH
+        nil
+      rescue Errno::EPERM
+        # Darwin may report EPERM for an already-disappeared process group.
+        begin
+          Process.kill(0, pid)
+        rescue Errno::ESRCH
+          return
+        end
+        raise
+      end
+
+      def cleanup_groups
+        groups = owned_groups.reject { |pid| pid == @pid }
+        groups.each { |pid| kill_group(pid) }
+        # Terminate every group before waiting for any adopted children.
+        groups.each do |pid| # rubocop:disable Style/CombinableLoops
+          loop { Process.waitpid(-pid) }
+        rescue Errno::ECHILD
+          nil
+        end
+        @status = @status.merge(masters: [])
+      end
+
+      def exited?
+        return true if @exit_status
+        return false unless @pid
+
+        pair = Process.waitpid2(@pid, Process::WNOHANG)
+        @exit_status = pair&.last
+        !@exit_status.nil?
+      end
+    end
+  end
+end

data/lib/gritz/worker/runner.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+require "io/wait"
+
+module Gritz
+  module Worker
+    # Owns transport resources and heartbeats in one serving process.
+    # @api private
+    class Runner
+      def initialize(index:, config:, logger:, status_io: nil, owner_channel: nil)
+        @index = index
+        @config = config
+        @logger = logger
+        @owner_channel = owner_channel
+        @status = owner_channel || (Supervisor::StatusChannel.new(status_io) if status_io)
+        @recorder = Metrics::Recorder.new
+        @metrics = Metrics::Aggregator.new
+        @metric_seq = 0
+        @born_at = monotonic
+      end
+
+      def run
+        @exit_status = 0
+        begin
+          @signals = Supervisor::SignalQueue.new(signals: %w[TERM INT QUIT HUP USR1 USR2])
+          report("booting")
+          require "gritz/native"
+          @boot_started = true
+          @config.preload! unless @config.preload_app?
+          @config.run_hooks(:on_worker_boot, @index)
+          router = Router.new(controllers: @config.controllers, strict: @config.strict_routes, logger: @logger)
+          dispatcher = Dispatcher.new(router:, middleware: @config.middleware, logger: @logger, metrics: @recorder,
+                                      worker: @index, log_format: @config.log_format, log_redact: @config.log_redact)
+          @adapter = Transport::Native.new(config: @config, dispatcher:, logger: @logger)
+          @port = @adapter.bind(@config.bind)
+          @adapter.start
+          unless @status
+            @admin = Supervisor::AdminServer.new(bind: @config.admin_bind, status: -> { snapshot }, ready: -> { ready? },
+                                                 metrics: -> { @metrics.render(workers: [@last_status].compact) }, logger: @logger)
+          end
+          @logger.info("Gritz #{Core::VERSION} listening on #{@config.bind.sub(/:\d+\z/, ":#{@port}")}")
+          report("ready")
+          serve
+        rescue StandardError, LoadError, SyntaxError, SystemExit => e
+          fail_worker(e)
+        ensure
+          finish
+        end
+        @exit_status
+      end
+
+      private
+
+      def serve
+        next_status = monotonic + @config.status_interval
+        loop do
+          raise ConfigurationError, "gRPC server stopped unexpectedly" unless @adapter.running?
+          raise ConfigurationError, "worker status pipe closed" if @status&.closed?
+
+          @signals.drain.each do |name|
+            case name
+            when "HUP" then @logger.reopen
+            when "USR1"
+              if @config.workers.zero?
+                @logger.warn("USR1 requires workers > 0; use USR2 to reload a single-process server")
+              else
+                drain
+              end
+            when "USR2"
+              if @owner_channel
+                @pending_reexec = true
+              else
+                @logger.warn("USR2 requires the gritz launcher")
+              end
+            when "QUIT"
+              drain
+              @adapter.kill
+              @transport_stopped = true
+              return 0
+            when "TERM", "INT"
+              drain
+              # Supervised children have already waited in the master's drain phase.
+              @drain_until ||= monotonic + (@config.workers.zero? ? @config.drain_delay : 0)
+            end
+          end
+          now = monotonic
+          if @drain_until && now >= @drain_until
+            @stop_deadline = now + @config.shutdown_timeout
+            @adapter.stop(deadline: Time.now + @config.shutdown_timeout)
+            @transport_stopped = true
+            return 0
+          end
+          @status&.flush
+          if @pending_reexec && @owner_channel.write(type: "reexec", pid: Process.pid)
+            @pending_reexec = false
+          end
+          publish_metrics
+          @admin&.poll
+          if now >= next_status
+            report(@draining ? "draining" : "ready")
+            next_status = now + @config.status_interval
+          end
+          next_event = [next_status, @drain_until].compact.min
+          IO.select([@signals.io, *@admin&.ios.to_a], nil, nil, (next_event - monotonic).clamp(0, 0.05))
+        end
+      end
+
+      def report(state, include_stats: true, **extra)
+        stats = include_stats && @adapter ? @adapter.stats : {}
+        @recorder.observe_rejected(stats.fetch(:rejected_total, 0)) if include_stats && @adapter
+        evaluate_health if state == "ready"
+        row = { inflight: 0, capacity: @config.threads, requests_total: 0, oldest_inflight_age: 0 }.merge(stats)
+        row.merge!(pid: Process.pid, index: @index, state:, ts: monotonic, port: @port,
+                   busy_threads: stats.fetch(:busy_threads, stats.fetch(:busy, 0)), healthy: state == "ready" && @healthy,
+                   checks: @checks || {}, worker_started_at: @born_at)
+        @last_status = row.merge(extra)
+        @status&.write(@owner_channel ? snapshot : @last_status)
+      end
+
+      def snapshot
+        { type: "status", pid: Process.pid, state: @draining ? "draining" : "running", desired: 1, workers: [@last_status].compact }
+      end
+
+      def ready? = !@draining && @last_status && @last_status[:state] == "ready" && @last_status[:healthy]
+
+      def evaluate_health
+        @checks = @config.health_checks.to_h do |name, callback|
+          [name, callback.call ? true : false]
+        rescue StandardError => e
+          @logger.warn("Health check #{name}: #{e.message}")
+          [name, false]
+        end
+        @healthy = @adapter.update_health(ready: true, checks: @checks)
+      end
+
+      def drain
+        return if @draining
+
+        @draining = true
+        @adapter.drain!
+        report("draining")
+      end
+
+      def publish_metrics
+        @pending_delta ||= @recorder.take_delta
+        return true unless @pending_delta
+
+        packet = { type: "metrics", pid: Process.pid, worker_pid: Process.pid, worker_started_at: @born_at,
+                   seq: @metric_seq + 1, delta: @pending_delta }
+        accepted = @status ? @status.write(packet) : @metrics.apply(self, packet)
+        if accepted
+          @metric_seq += 1
+          @pending_delta = nil
+        end
+        accepted
+      end
+
+      def fail_worker(error)
+        @exit_status = 1
+        @logger.error(error.full_message)
+        report("failed", include_stats: false, error: error.message[0, 512])
+      end
+
+      def finish
+        cleanup { @adapter&.kill unless @transport_stopped }
+        cleanup { @config.run_hooks(:on_worker_shutdown, @index) if @boot_started }
+        cleanup { @recorder.observe_rejected(@adapter.stats.fetch(:rejected_total, 0)) if @adapter }
+        deadline = @stop_deadline || (monotonic + @config.shutdown_timeout)
+        loop do
+          @status&.flush
+          accepted = publish_metrics
+          @pending_delta ||= @recorder.take_delta if accepted
+          break if accepted && !@pending_delta && (@status.nil? || @status.flush)
+          break if monotonic >= deadline || @status&.closed?
+
+          @status&.io&.wait_writable((deadline - monotonic).clamp(0, 0.01))
+        end
+        until report("stopped", include_stats: false, exit_status: @exit_status) != false
+          break if monotonic >= deadline || @status&.closed?
+
+          @status.flush
+          @status.io.wait_writable((deadline - monotonic).clamp(0, 0.01)) unless @status.closed?
+        end
+        @status.io.wait_writable((deadline - monotonic).clamp(0, 0.01)) until !@status || @status.flush || @status.closed? || monotonic >= deadline
+      ensure
+        begin
+          @signals&.close
+          @admin&.close
+        ensure
+          @status&.close
+        end
+      end
+
+      def cleanup
+        yield
+      rescue StandardError, LoadError, SyntaxError, SystemExit => e
+        fail_worker(e)
+      end
+
+      def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gritz-core
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Yudai Takada
@@ -9,6 +9,26 @@ bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: fiddle
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: google-protobuf
   requirement: !ruby/object:Gem::Requirement
@@ -111,17 +131,29 @@ files:
 - lib/gritz/dispatcher.rb
 - lib/gritz/dsl.rb
 - lib/gritz/errors.rb
+- lib/gritz/fork_guard.rb
 - lib/gritz/method_descriptor.rb
+- lib/gritz/metrics/aggregator.rb
+- lib/gritz/metrics/recorder.rb
 - lib/gritz/middleware/context.rb
 - lib/gritz/middleware/exception_mapper.rb
 - lib/gritz/middleware/logging.rb
+- lib/gritz/middleware/metrics.rb
 - lib/gritz/middleware/request_id.rb
 - lib/gritz/middleware/stack.rb
 - lib/gritz/router.rb
+- lib/gritz/supervisor/admin_server.rb
+- lib/gritz/supervisor/launcher.rb
+- lib/gritz/supervisor/master.rb
+- lib/gritz/supervisor/signal_queue.rb
+- lib/gritz/supervisor/status_channel.rb
+- lib/gritz/supervisor/worker_handle.rb
+- lib/gritz/testing/cluster.rb
 - lib/gritz/testing/in_memory_call.rb
 - lib/gritz/testing/minitest.rb
 - lib/gritz/testing/rpc_helper.rb
 - lib/gritz/testing/rspec.rb
+- lib/gritz/worker/runner.rb
 homepage: https://github.com/gritzrpc/gritz-core
 licenses:
 - MIT
@@ -144,7 +176,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.16
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Transport-independent controllers and middleware for Gritz
 test_files: []