gritz-core 0.9.0 → 0.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 40b995a6a11953b9aec90291e60bbd928911c1e4e0337a7241daf3c34b210e27
-  data.tar.gz: 511df3afe2b9071e7920b36633a58fdd759832ccf158392a7e2be553385a72f6
+  metadata.gz: b938dbec401c131416b1a0a0b0f49fa9b140bf1567d8738f8f29d1c69dfa3ec1
+  data.tar.gz: df14f03e83c2ee4c72e62e7d7cedb9eab105541d41d9aae9ed2c71e929e7951c
 SHA512:
-  metadata.gz: a66f9c4d58aca49a6e4bf9912b2f2ecf1f0391e2381cb9c3b9cbe29ab7bb47c777fcad4667afb67f90a8d2d98f7fc9df231cf82a4fa25021777b264e074ac6d4
-  data.tar.gz: c1bbe9d8f404c15669814fab6f85d3b6b389b0ff3bd7548cfb5ce5e7ca98e6672d8811ca9707339c47a09128aa06b4475606e386104d06f52403098400073953
+  metadata.gz: 453162303ec6b73e2dba7aa2cbd878c86b3afd3d69ac2641d8f0813abf8a27e5c66171ffa50eabfbb35e5cfcc591d0c99e0783a897c214fa2bc38082366a27e5
+  data.tar.gz: b84efb28c932187519075a5ad79016cd1322650deaef419f1dccf091a557cb328b88a8b988ec2855fd0efd0c012574e0ccd775f97b935f9053b47ce0980d6253

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.9.1
+
+- Add `gritz stats`, `stop` and `restart` to inspect the running server and signal its lifecycle owner without executing application configuration.
+- Honor configured `reexec_timeout` values above 60 seconds for initial and replacement master readiness, including time already spent starting the generation.
+- Check cooperative cancellation and pool-slot recovery for all four RPC forms in the shared transport contract.
+
 ## 0.9.0
 
 - Start the 0.9 stabilization series with the documented public API and support policy.

data/lib/gritz/cli.rb

@@ -4,9 +4,11 @@ require "optparse"
 require "logger"
 require "rbconfig"
 require "socket"
+require "net/http"
+require "json"
 
 module Gritz
-  # Starts a server, lists routes, or checks configuration and fork safety.
+  # Starts and operates servers, lists routes, or checks configuration and fork safety.
   # @api public
   class CLI
     CHILD_BOOTSTRAP = <<~RUBY
@@ -35,11 +37,13 @@ module Gritz
       path = nil
       overrides = {}
       parser = OptionParser.new do |options|
-        options.banner = "Usage: gritz [start|routes|check] [-C config/gritz.rb] [options]"
+        options.banner = "Usage: gritz [start|routes|check|stats|stop|restart] [-C config/gritz.rb] [options]"
         options.on("-C", "--config PATH", "Configuration file") { |value| path = value }
         options.on("--workers N", Integer) { |value| overrides[:workers] = value }
         options.on("--threads N", Integer) { |value| overrides[:threads] = value }
         options.on("--bind ADDRESS") { |value| overrides[:bind] = value }
+        options.on("--admin-bind ADDRESS", "Admin HTTP host:port (operations do not load config)") { |value| overrides[:admin_bind] = value }
+        options.on("--pid-file PATH", "Active master PID file; operations fall back if Admin is unavailable") { |value| overrides[:pid_file] = value }
         options.on("--strict-routes") { overrides[:strict_routes] = true }
         options.on("-v", "--version") {
           @stdout.puts(Core::VERSION)
@@ -52,9 +56,15 @@ module Gritz
       end
       parser.parse!(args)
       command = args.shift || "start"
-      raise ConfigurationError, "Unknown command #{command}" unless %w[start routes check].include?(command)
+      raise ConfigurationError, "Unknown command #{command}" unless %w[start routes check stats stop restart].include?(command)
       raise ConfigurationError, "Unexpected argument #{args.first}" unless args.empty?
 
+      if %w[stats stop restart].include?(command)
+        raise ConfigurationError, "#{command} does not load configuration; use --admin-bind or --pid-file instead of -C" if path
+
+        return run_operation(command, overrides)
+      end
+
       logger = Logger.new(@stdout)
       logger.formatter = ->(_severity, _time, _progname, message) { "#{message}\n" }
       if command == "start" && @launch && !@owner_channel
@@ -127,5 +137,116 @@ module Gritz
       @owner_channel&.close
       listener&.close unless listener&.closed?
     end
+
+    private
+
+    def run_operation(command, overrides)
+      address = overrides.fetch(:admin_bind) { @env.fetch("GRITZ_ADMIN_BIND", Configuration::DEFAULTS[:admin_bind]) }
+      path = overrides.fetch(:pid_file) { @env.fetch("GRITZ_PID_FILE", "") }
+      file_pid = read_pid_file(path) unless path.empty? || command == "stats"
+      status = read_admin_status(address, fallback: !file_pid.nil?)
+      if command == "stats"
+        @stdout.puts "State: #{status.fetch('state')}  Master PID: #{status.fetch('pid')}  Owner PID: #{status['owner_pid'] || 'n/a'}"
+        @stdout.puts "WORKER  PID  STATE  RSS  PSS"
+        status.fetch("workers").each do |worker|
+          @stdout.puts "#{worker['index']}  #{worker.fetch('pid')}  #{worker.fetch('state')}  " \
+                       "#{format_bytes(worker['rss_bytes'])}  #{format_bytes(worker['pss_bytes'])}"
+        end
+      else
+        if status && file_pid && status.fetch("pid") != file_pid
+          raise ConfigurationError, "Admin master PID #{status.fetch('pid')} does not match PID file #{file_pid}"
+        end
+
+        pid = status ? status.fetch("owner_pid", status.fetch("pid")) : file_pid
+        signal = command == "stop" ? "TERM" : "USR2"
+        begin
+          Process.kill(signal, pid)
+        rescue SystemCallError => e
+          raise ConfigurationError, "Cannot send #{signal} to PID #{pid}: #{e.message}"
+        end
+        @stdout.puts "Sent #{signal} to PID #{pid}"
+      end
+      0
+    end
+
+    def read_pid_file(path)
+      File.open(path, File::RDONLY | File::NONBLOCK) do |file|
+        raise ConfigurationError, "PID file must be a regular file: #{path}" unless file.stat.file?
+        raise ConfigurationError, "Invalid PID file: #{path}" if file.stat.size > 64
+
+        value = file.read(65).to_s.strip
+        raise ConfigurationError, "Invalid PID file: #{path}" unless value.match?(/\A[0-9]{1,10}\z/)
+
+        validate_pid!(Integer(value, 10))
+      end
+    rescue SystemCallError => e
+      raise ConfigurationError, "Cannot read PID file #{path}: #{e.message}"
+    end
+
+    def read_admin_status(address, fallback:)
+      match = address.match(/\A(\[[a-zA-Z0-9_.:%-]+\]|[a-zA-Z0-9_.-]+):([0-9]+)\z/)
+      raise ConfigurationError, "Invalid Admin address: #{address}" unless match && Integer(match[2], 10).between?(1, 65_535)
+
+      # Operations always target the requested endpoint, never an environment HTTP proxy.
+      http = Net::HTTP.new(match[1].delete_prefix("[").delete_suffix("]"), Integer(match[2], 10), nil)
+      http.open_timeout = http.read_timeout = 2
+      http.max_retries = 0
+      body = +""
+      response_started = false
+      http.start do |client|
+        client.request(Net::HTTP::Get.new("/status")) do |response|
+          response_started = true
+          raise ConfigurationError, "Admin /status returned HTTP #{response.code}" unless response.code == "200"
+
+          response.ignore_eof = false
+          response.read_body do |chunk|
+            raise ConfigurationError, "Admin /status response exceeds 1 MiB" if body.bytesize + chunk.bytesize > 1_048_576
+
+            body << chunk
+          end
+        end
+      end
+      validate_status!(JSON.parse(body))
+    rescue SystemCallError, IOError, SocketError, Timeout::Error => e
+      return nil if fallback && !response_started
+
+      raise ConfigurationError, "Admin /status unavailable at #{address}: #{e.message}; stop/restart can use --pid-file"
+    rescue JSON::ParserError, Net::ProtocolError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError => e
+      raise ConfigurationError, "Invalid Admin /status response: #{e.message}"
+    end
+
+    def validate_pid!(pid)
+      unless pid.is_a?(Integer) && pid.between?(2, 2_147_483_647) && pid != Process.pid
+        raise ConfigurationError, "Invalid server PID: #{pid.inspect}"
+      end
+
+      pid
+    end
+
+    def validate_status!(status)
+      unless status.is_a?(Hash) && %w[starting running draining].include?(status["state"]) && status["workers"].is_a?(Array)
+        raise ConfigurationError, "Invalid Admin /status process state"
+      end
+
+      validate_pid!(status["pid"])
+      validate_pid!(status["owner_pid"]) if status.key?("owner_pid")
+      status["workers"].each do |worker|
+        unless worker.is_a?(Hash) && Supervisor::WorkerHandle::STATES.include?(worker["state"]) &&
+               worker["index"].is_a?(Integer) && worker["index"] >= 0
+          raise ConfigurationError, "Invalid Admin /status worker state"
+        end
+
+        validate_pid!(worker["pid"])
+        %w[rss_bytes pss_bytes].each do |name|
+          value = worker[name]
+          unless value.nil? || (value.is_a?(Numeric) && value.real? && value.finite? && value >= 0)
+            raise ConfigurationError, "Invalid Admin /status #{name}"
+          end
+        end
+      end
+      status
+    end
+
+    def format_bytes(value) = value ? format("%.2f MiB", value / (1024.0 * 1024)) : "n/a"
   end
 end

data/lib/gritz/core/version.rb

@@ -2,6 +2,6 @@
 
 module Gritz
   module Core
-    VERSION = "0.9.0"
+    VERSION = "0.9.1"
   end
 end

data/lib/gritz/supervisor/launcher.rb

@@ -9,7 +9,7 @@ module Gritz
     # Keeps process ownership and probes stable while fresh master interpreters replace each other.
     # @api private
     class Launcher
-      Generation = Struct.new(:pid, :channel, :token, :role, :metadata, :snapshot, :deadline,
+      Generation = Struct.new(:pid, :channel, :token, :role, :metadata, :snapshot, :started_at, :deadline,
                               :kill_at, :kill_sent, :exit_status, :group_reaped, :identities, keyword_init: true)
       SIGNALS = %w[TERM INT QUIT USR1 USR2 TTIN TTOU HUP CHLD].freeze
 
@@ -91,11 +91,12 @@ module Gritz
 
       def spawn_generation
         parent, child = UNIXSocket.pair
+        started_at = monotonic
         pid = Process.spawn(@env.to_h.merge("GRITZ_INTERNAL_OWNER_FD" => "3"), *@command,
                             3 => child, out: @stdout, err: @stderr, pgroup: true)
         @serial += 1
         @candidate = Generation.new(pid: pid, channel: StatusChannel.new(parent), token: @serial, role: :candidate,
-                                    deadline: monotonic + @startup_timeout, identities: {})
+                                    started_at:, deadline: started_at + @startup_timeout, identities: {})
         @generations[pid] = @candidate
         @reexec = { state: "starting", pid: pid } if @active
       rescue StandardError
@@ -137,6 +138,7 @@ module Gritz
 
       def configure_generation(generation, metadata)
         return if generation.metadata || @stopping
+        raise ConfigurationError, "replacement master configuration timed out" if monotonic >= generation.deadline
 
         if metadata[:listener_strategy] && !%w[reuseport inherited_fd].include?(metadata[:listener_strategy])
           raise ConfigurationError, "Invalid master listener_strategy"
@@ -163,7 +165,7 @@ module Gritz
           lock_pid_file(metadata[:pid_file]) unless metadata[:pid_file].empty?
         end
         generation.metadata = metadata
-        generation.deadline = [generation.deadline, monotonic + metadata[:reexec_timeout]].min
+        generation.deadline = generation.started_at + metadata[:reexec_timeout]
         if metadata[:listener_strategy] == "inherited_fd"
           @listener ||= Listener.bind(metadata[:bind])
           generation.channel.io.send_io(@listener)

data/lib/gritz/testing/transport_contract.rb

@@ -218,6 +218,58 @@ RSpec.shared_examples Gritz::Testing::TransportContract do |adapter:, service:,
     end
   end
 
+  %i[say_hello record_names list_greetings chat].each do |action|
+    it "lets a cooperative #{action} controller observe cancellation and releases its serving slot" do
+      entered = Queue.new
+      completed = Queue.new
+      release = Queue.new
+      response = contract_reply
+      controller = contract_controller do
+        define_method(action) do
+          observing = self.request.message.name == "cancel"
+          if observing
+            entered << true
+            context.check_cancelled! until release.pop(timeout: 0.005)
+          end
+          result = response.new(message: "available")
+          context.method.server_streaming? ? stream.write(result) : result
+        ensure
+          completed << true if observing
+        end
+      end
+      with_contract_server(controller, threads: 1) do |client, server|
+        streaming_input = %i[record_names chat].include?(action)
+        message = contract_request.new(name: "cancel")
+        operation = client.public_send(action, streaming_input ? [message] : message, return_op: true, deadline: Time.now + 5)
+        caller = Thread.new do
+          result = operation.execute
+          result.to_a if %i[list_greetings chat].include?(action)
+        rescue GRPC::BadStatus => e
+          e
+        end
+        expect(entered.pop(timeout: 2)).to be(true)
+        operation.cancel
+        expect(caller.join(2)).not_to be_nil
+        expect(caller.value).to be_a(GRPC::Cancelled)
+        expect(completed.pop(timeout: 1)).to be(true)
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 1
+        until server.transport.stats[:inflight].zero? && (adapter == :async || server.transport.stats[:busy].zero?)
+          raise "cancelled controller retained its serving slot" if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+
+          sleep 0.005
+        end
+        message = contract_request.new
+        result = client.public_send(action, streaming_input ? [message] : message, deadline: Time.now + 1)
+        result = result.to_a.first if %i[list_greetings chat].include?(action)
+        expect(result.message).to eq("available")
+      ensure
+        release << true
+        operation&.cancel
+        caller&.join(2)
+      end
+    end
+  end
+
   it "finishes an in-flight response during graceful shutdown" do
     entered = Queue.new
     gate = Queue.new

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gritz-core
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.9.1
 platform: ruby
 authors:
 - Yudai Takada