gritz-core 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9d69bbe23da7b5ec3c973bdbe3a2c2c1d1596b6632695f446ae6fb242d0d1f94
-  data.tar.gz: 89929d525c1709e164057aa7d684317db596734395371cf942019832cf7189bd
+  metadata.gz: 03bd5114b9267c0bc0b5daa80d14433d4941c753321338fc5b83b2281f4d6bf1
+  data.tar.gz: 67fabc1d94c5cca35149679a2e582c983a88122a5299638d05c639387ea48fa4
 SHA512:
-  metadata.gz: c4c61c0a88eedb466c3f556f9cd9861468f496b0ba6303be58133d58c6761d8d6fa6a1a9c4b51e7dd6bec168b8207aa8b82562d096f0ead5f8745e351a67c790
-  data.tar.gz: 5e7d9a717648f45c0f3774f3de35511608d6b6fa7fc5fc6f6be61fbf4d7b874c1cfafdd348b9441e0f37a528a30d341f3eddfa45aa50760255a42f7b644a58cf
+  metadata.gz: 22ba419be6cd7bf1754eaccc7af3bde09ac0ca9dcac494f5d1932306aa73d595e0861b1c9ff1af9e694195f86d6c19e3103161a721ccd8e3f4c88ae72c93c56d
+  data.tar.gz: 1986b1cc2c4a32d0baf218b8268b1ced8399931c1a48379f46071f891c09d4c89cfe489fb707683dcf0d4ed239c708b3bfdd36a156efb0696f14d1764a10eeb6

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 0.3.0
+
+- Replace workers one at a time with `USR1` and recycle them by request count, memory or lifetime.
+- Reload application code with `USR2`; retain the serving master if replacement startup fails, and update the PID file after readiness.
+- Keep admin probes and Prometheus totals available across worker and master replacement. Add application health checks and per-worker status.
+- Emit one structured completion log per RPC with JSON or logfmt output and credential redaction.
+
+## 0.2.0
+
+- Supervise forked workers with boot and heartbeat timeouts, automatic replacement, graceful shutdown, lifecycle hooks, dynamic worker counts and log reopening.
+- Detect unsafe master-side gRPC initialization with ForkGuard and `gritz check`.
+- Add `Gritz::Testing::Cluster` for process lifecycle integration tests.
+
 ## 0.1.0
 
 Initial release.

data/README.md

@@ -1,6 +1,6 @@
 # Gritz Core
 
-Transport-independent controllers, routing, middleware, configuration and network-free testing for Ruby gRPC applications. Requires CRuby 3.3 or later. This gem has no grpc dependency and never loads a transport by itself.
+Transport-independent controllers, routing, middleware, configuration, forked-worker supervision and network-free testing for Ruby gRPC applications. Requires CRuby 3.3 or later. This gem has no grpc dependency. Requiring the core does not load a transport; starting a server loads the selected adapter.
 
 ```ruby
 require "gritz/core"
@@ -10,6 +10,8 @@ Use [gritz](https://github.com/gritzrpc/gritz) for the default framework and exe
 
 Controller tests can load `gritz/testing/rspec` or `gritz/testing/minitest` and run without a network or the grpc gem. See the [controller examples](https://github.com/gritzrpc/gritz#controllers) and [configuration guide](docs/guides/configuration.md).
 
+Supervised servers support phased replacement, fresh-interpreter hot reexec, request/memory/lifetime recycling, Admin HTTP probes and process-wide Prometheus totals. The CLI keeps one lifecycle owner across master generations; application controllers and gRPC resources stay in serving processes.
+
 ## Development
 
 ```sh

data/lib/gritz/cli.rb

@@ -2,15 +2,31 @@
 
 require "optparse"
 require "logger"
+require "rbconfig"
+require "socket"
 
 module Gritz
-  # Command line entry point for the single-process server and route listing.
+  # Starts a server, lists routes, or checks configuration and fork safety.
   # @api public
   class CLI
-    def initialize(stdout: $stdout, stderr: $stderr, env: ENV)
+    CHILD_BOOTSTRAP = <<~RUBY
+      require "socket"
+      fd = ENV.delete("GRITZ_INTERNAL_OWNER_FD")
+      exit Gritz::CLI.new(owner_io: UNIXSocket.for_fd(Integer(fd))).run(ARGV)
+    RUBY
+
+    def self.child_command(argv)
+      [RbConfig.ruby, "-I", $LOAD_PATH.join(File::PATH_SEPARATOR), "-rgritz/core", "-e", CHILD_BOOTSTRAP, "--", *argv]
+    end
+
+    def initialize(stdout: $stdout, stderr: $stderr, env: ENV, status_io: nil, owner_io: nil, launch: false, launch_command: nil)
       @stdout = stdout
       @stderr = stderr
       @env = env
+      @status_io = status_io
+      @owner_channel = Supervisor::StatusChannel.new(owner_io) if owner_io
+      @launch = launch
+      @launch_command = launch_command
     end
 
     def run(argv)
@@ -18,7 +34,7 @@ module Gritz
       path = nil
       overrides = {}
       parser = OptionParser.new do |options|
-        options.banner = "Usage: gritz [start|routes] [-C config/gritz.rb] [options]"
+        options.banner = "Usage: gritz [start|routes|check] [-C config/gritz.rb] [options]"
         options.on("-C", "--config PATH", "Configuration file") { |value| path = value }
         options.on("--workers N", Integer) { |value| overrides[:workers] = value }
         options.on("--threads N", Integer) { |value| overrides[:threads] = value }
@@ -35,52 +51,62 @@ module Gritz
       end
       parser.parse!(args)
       command = args.shift || "start"
-      raise ConfigurationError, "Unknown command #{command}" unless %w[start routes].include?(command)
+      raise ConfigurationError, "Unknown command #{command}" unless %w[start routes check].include?(command)
       raise ConfigurationError, "Unexpected argument #{args.first}" unless args.empty?
 
-      path ||= "config/gritz.rb" if File.file?("config/gritz.rb")
-      config = Configuration.load(path: path, env: @env, overrides: overrides)
-      config.validate_single_process! if command == "start"
-      config.preload! if config.preload_app?
       logger = Logger.new(@stdout)
       logger.formatter = ->(_severity, _time, _progname, message) { "#{message}\n" }
-      router = Router.new(controllers: config.controllers, strict: config.strict_routes, logger: logger)
+      if command == "start" && @launch && !@owner_channel
+        return Supervisor::Launcher.new(command: @launch_command || self.class.child_command(argv), env: @env,
+                                        stdout: @stdout, stderr: @stderr, logger: logger, status_io: @status_io).run
+      end
+
+      path ||= "config/gritz.rb" if File.file?("config/gritz.rb")
+      guard = ForkGuard.activate(mode: :record) if command != "routes"
+      require "gritz/native" if command != "routes"
+      config = Configuration.load(path: path, env: @env, overrides: overrides)
+      if command == "check"
+        config.preload! if config.preload_app?
+        guard.violations.each { |violation| @stderr.puts(violation.message) }
+        return 1 unless guard.violations.empty?
+
+        config.validate_runtime!
+        Router.new(controllers: config.controllers, strict: config.strict_routes, logger: logger)
+        @stdout.puts "Configuration and fork safety checks passed"
+      elsif command == "start"
+        config.validate_runtime!
+        if config.workers.positive? && config.fork_mode == :clean
+          raise guard.violations.first if config.fork_guard == :raise && !guard.violations.empty?
+
+          guard.violations.each { |violation| logger.warn(violation.message) } if config.fork_guard == :warn
+        end
+        @stdout.sync = true if @stdout.respond_to?(:sync=)
+        @owner_channel&.write(type: "configured", pid: Process.pid, workers: config.workers, bind: config.bind,
+                              admin_bind: config.admin_bind, min_ready_workers: config.min_ready_workers, pid_file: config.pid_file,
+                              reexec_timeout: config.reexec_timeout, drain_delay: config.drain_delay, shutdown_timeout: config.shutdown_timeout)
+        if config.workers.positive?
+          return Supervisor::Master.new(config, logger: logger, status_io: @status_io, owner_channel: @owner_channel).run
+        end
+
+        ForkGuard.deactivate
+        config.preload! if config.preload_app?
+        return Worker::Runner.new(index: 0, config: config, logger: logger, owner_channel: @owner_channel).run
+      else
+        config.preload! if config.preload_app?
+        router = Router.new(controllers: config.controllers, strict: config.strict_routes, logger: logger)
+      end
       if command == "routes"
         router.routes.each_value do |route|
           @stdout.puts "#{route.full_name} #{route.kind} #{route.controller}##{route.action}"
         end
-      else
-        start(config, router, logger)
       end
       0
-    rescue ConfigurationError, OptionParser::ParseError, ArgumentError, Errno::ENOENT, SyntaxError, LoadError => e
+    rescue ConfigurationError, ForkGuard::Violation, ArgumentError, Errno::ENOENT, SyntaxError, LoadError, RuntimeError => e
       @stderr.puts "gritz: #{e.message}"
       1
-    end
-
-    private
-
-    def start(config, router, logger)
-      @stdout.sync = true if @stdout.respond_to?(:sync=)
-      # Load the adapter only when starting; route inspection stays transport-independent.
-      require "gritz/native"
-      dispatcher = Dispatcher.new(router: router, middleware: config.middleware, logger: logger)
-      adapter = Transport::Native.new(config: config, dispatcher: dispatcher, logger: logger)
-      signals = []
-      previous = %w[TERM INT QUIT].to_h { |name| [name, Signal.trap(name) { signals << name }] }
-      config.run_hooks(:on_worker_boot, 0)
-      port = adapter.bind(config.bind)
-      adapter.start
-      logger.info("Gritz #{Core::VERSION} listening on #{config.bind.sub(/:\d+\z/, ":#{port}")}")
-      while signals.empty?
-        sleep 0.05
-        raise ConfigurationError, "gRPC server stopped unexpectedly" unless adapter.running?
-      end
-      signals.shift == "QUIT" ? adapter.kill : adapter.stop(deadline: Time.now + config.shutdown_timeout)
     ensure
-      previous&.each { |name, handler| Signal.trap(name, handler) }
-      adapter&.kill if adapter&.running?
-      config.run_hooks(:on_worker_shutdown, 0)
+      ForkGuard.deactivate if guard
+      @owner_channel&.close
     end
   end
 end

data/lib/gritz/configuration.rb

@@ -3,7 +3,7 @@
 require "json"
 
 module Gritz
-  # Startup settings. Process-related settings are reserved for the supervisor.
+  # Validated startup settings for single-process and supervised servers.
   # @api public
   class Configuration
     DEFAULTS = {
@@ -18,7 +18,8 @@ module Gritz
       keepalive_permit_without_calls: true,
       max_receive_message_size: 4 * 1024 * 1024,
       max_send_message_size: 4 * 1024 * 1024, max_metadata_size: 8192,
-      metrics_backend: :pipe, log_format: :json, worker_recycle: {}, tls: {}
+      metrics_backend: :pipe, log_format: :json, log_redact: [], worker_recycle: {}, tls: {},
+      pid_file: "", reexec_timeout: 60.0
     }.freeze
     ENUMS = {
       transport: %i[native async], listener_strategy: %i[reuseport inherited_fd port_per_worker],
@@ -46,6 +47,7 @@ module Gritz
       DSL.new(config).evaluate(path) if path
       env.each do |key, value|
         next unless key.start_with?("GRITZ_")
+        next if key == "GRITZ_RELEASE" # Dependency selection in the release workflow, not a runtime setting.
 
         name = key.delete_prefix("GRITZ_").downcase.to_sym
         raise ConfigurationError, "Unknown environment setting #{key}" unless DEFAULTS.key?(name)
@@ -62,7 +64,7 @@ module Gritz
       when Integer then Integer(value, 10)
       when Float then Float(value)
       when Symbol then value.to_sym
-      when Hash then JSON.parse(value, symbolize_names: true)
+      when Hash, Array then JSON.parse(value, symbolize_names: true)
       when true, false
         return true if %w[true 1].include?(value)
         return false if %w[false 0].include?(value)
@@ -85,7 +87,7 @@ module Gritz
                 end
         raise ConfigurationError, "Invalid #{name}: #{value.inspect}" unless valid
       end
-      %i[shutdown_timeout worker_boot_timeout worker_timeout status_interval].each do |name|
+      %i[shutdown_timeout worker_boot_timeout worker_timeout status_interval reexec_timeout].each do |name|
         raise ConfigurationError, "#{name} must be positive" unless public_send(name).positive?
       end
       ENUMS.each do |name, allowed|
@@ -97,6 +99,12 @@ module Gritz
       raise ConfigurationError, "controllers must be an Array of classes" unless controllers.is_a?(Array) && controllers.all?(Class)
       raise ConfigurationError, "middleware must be a Stack" unless middleware.is_a?(Middleware::Stack)
       raise ConfigurationError, "preload_app must be boolean" unless [true, false].include?(preload_app)
+      unless log_redact.all? { |name| name.is_a?(String) || name.is_a?(Symbol) }
+        raise ConfigurationError, "log_redact must contain metadata names"
+      end
+      unless health_checks.all? { |name, check| (name.is_a?(String) || name.is_a?(Symbol)) && check.respond_to?(:call) }
+        raise ConfigurationError, "health_checks must contain named callbacks"
+      end
 
       self
     end
@@ -104,22 +112,30 @@ module Gritz
     def preload_app? = preload_app
 
     # Fail before allocating transport resources for unsupported release features.
-    def validate_single_process!
+    def validate_runtime!
       validate!
-      raise ConfigurationError, "v0.1 supports workers 0; the supervisor is planned for v0.2" unless workers.zero?
-      raise ConfigurationError, "v0.1 supports transport :native" unless transport == :native
-      raise ConfigurationError, "v0.1 supports listener_strategy :reuseport" unless listener_strategy == :reuseport
-      raise ConfigurationError, "TLS is planned for v0.3" unless tls.empty?
-      raise ConfigurationError, "worker_recycle requires the supervisor" unless worker_recycle.empty?
-      raise ConfigurationError, "Health checks are planned for v0.3" unless health_checks.empty?
-      raise ConfigurationError, "v0.1 supports log_format :json" unless log_format == :json
-      raise ConfigurationError, "v0.1 reserves metrics_backend :pipe; metrics export is planned for v0.3" unless metrics_backend == :pipe
-      raise ConfigurationError, "grpc_fork_support requires the supervisor" unless fork_mode == :clean
+      raise ConfigurationError, "This release supports transport :native" unless transport == :native
+      raise ConfigurationError, "This release supports listener_strategy :reuseport" unless listener_strategy == :reuseport
+      raise ConfigurationError, "This release supports metrics_backend :pipe" unless metrics_backend == :pipe
+      raise ConfigurationError, "This release supports phased_restart_surge 1" unless phased_restart_surge == 1
+
+      unless worker_recycle.empty?
+        raise ConfigurationError, "worker_recycle requires workers > 0" unless workers.positive?
+        raise ConfigurationError, "worker_recycle requires a fixed bind port" if bind.end_with?(":0")
+      end
+      raise ConfigurationError, "grpc_fork_support requires workers > 0" if workers.zero? && fork_mode != :clean
       raise ConfigurationError, "at least one controller must be registered" if controllers.empty?
 
       self
     end
 
+    def validate_single_process!
+      validate_runtime!
+      raise ConfigurationError, "Testing::Server requires workers 0; use Testing::Cluster for supervised servers" unless workers.zero?
+
+      self
+    end
+
     def add_preloader(&block)
       @preload_app = true
       @preloaders << block if block

data/lib/gritz/context.rb

@@ -4,13 +4,19 @@ module Gritz
   # Request state inherited by child fibers and threads using Ruby Fiber storage.
   # @api public
   class Context
-    attr_reader :call, :logger, :store
+    attr_reader :call, :logger, :store, :metrics, :worker, :log_format, :log_redact,
+                :requests_count, :responses_count, :bytes_in, :bytes_out
     attr_accessor :request_id, :trace_context
 
-    def initialize(call:, logger:)
+    def initialize(call:, logger:, metrics: nil, worker: nil, log_format: :json, log_redact: [])
       @call = call
       @logger = logger
       @store = {}
+      @metrics = metrics
+      @worker = worker
+      @log_format = log_format
+      @log_redact = log_redact.map(&:to_s)
+      @requests_count = @responses_count = @bytes_in = @bytes_out = 0
     end
 
     def self.current = Fiber[:gritz_context]
@@ -46,5 +52,28 @@ module Gritz
 
       message
     end
+
+    # @api private
+    def record_received(message)
+      return unless message
+
+      size = encoded_size(message)
+      @requests_count += 1
+      @bytes_in += size
+    end
+
+    # @api private
+    def record_sent(message)
+      size = encoded_size(message)
+      @responses_count += 1
+      @bytes_out += size
+    end
+
+    private
+
+    def encoded_size(message)
+      # ponytail: byte accounting encodes again; instrument native serialization if the Phase 6 overhead budget is exceeded.
+      message.class.respond_to?(:encode) ? message.class.encode(message).bytesize : message.to_s.bytesize
+    end
   end
 end

data/lib/gritz/controller.rb

@@ -96,6 +96,10 @@ module Gritz
       def initialize(call, context)
         @call = call
         @context = context
+        unless context.method.client_streaming?
+          @message = @call.read
+          @context.record_received(@message)
+        end
       end
 
       def message
@@ -104,14 +108,23 @@ module Gritz
         return @message if defined?(@message)
 
         @message = @call.read
+        @context.record_received(@message)
+        @message
       end
 
       def each_message
         return enum_for(__method__) unless block_given?
 
+        unless @context.method.client_streaming?
+          value = message
+          yield value if value
+          return
+        end
+
         @call.each_message do |message|
           @context.check_deadline!
           @context.check_cancelled!
+          @context.record_received(message)
           yield message
         end
       end
@@ -133,6 +146,7 @@ module Gritz
         @context.check_cancelled!
         @context.validate_response!(message)
         @call.write(message)
+        @context.record_sent(message)
       end
     end
   end

data/lib/gritz/core/version.rb

@@ -2,6 +2,6 @@
 
 module Gritz
   module Core
-    VERSION = "0.1.0"
+    VERSION = "0.3.0"
   end
 end

data/lib/gritz/dispatcher.rb

@@ -8,9 +8,11 @@ module Gritz
   class Dispatcher
     attr_reader :router
 
-    def initialize(router:, middleware: Middleware::Stack.default, logger: Logger.new($stdout))
+    def initialize(router:, middleware: Middleware::Stack.default, logger: Logger.new($stdout),
+                   metrics: nil, worker: nil, log_format: :json, log_redact: [])
       @router = router
       @logger = logger
+      @context_options = { metrics: metrics, worker: worker, log_format: log_format, log_redact: log_redact }
       endpoint = lambda do |context|
         context.check_deadline!
         context.check_cancelled!
@@ -18,13 +20,17 @@ module Gritz
         result = descriptor.controller.new(context:).process(descriptor.action)
         context.check_deadline!
         context.check_cancelled!
-        descriptor.server_streaming? ? result : context.validate_response!(result)
+        unless descriptor.server_streaming?
+          context.validate_response!(result)
+          context.record_sent(result)
+        end
+        result
       end
       @app = middleware ? middleware.build(endpoint) : endpoint
     end
 
     def call(call)
-      @app.call(Context.new(call:, logger: @logger))
+      @app.call(Context.new(call:, logger: @logger, **@context_options))
     end
   end
 end

data/lib/gritz/fork_guard.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Gritz
+  # Detects transport resource constructors in the master before workers fork.
+  # @api public
+  class ForkGuard
+    class Violation < StandardError
+      attr_reader :klass, :locations
+
+      def initialize(klass, locations)
+        @klass = klass
+        @locations = locations
+        super("gRPC object was created in the master process before fork.\n  " \
+              "#{klass}.new\n#{locations.map { |location| "    from #{location}" }.join("\n")}\n" \
+              "Fix: create it in an on_worker_boot hook.")
+      end
+    end
+
+    module ConstructorHook
+      def new(...)
+        Gritz::ForkGuard.check!(self)
+        super
+      end
+    end
+
+    attr_reader :mode, :logger, :master_pid, :violations
+
+    class << self
+      attr_reader :current
+
+      def activate(mode: :raise, logger: nil, master_pid: Process.pid)
+        @current = new(mode:, logger:, master_pid:)
+      end
+
+      def deactivate
+        @current = nil
+      end
+
+      def install(klass)
+        klass.singleton_class.prepend(ConstructorHook) unless klass.singleton_class.ancestors.include?(ConstructorHook)
+      end
+
+      def master?
+        @current&.master? || false
+      end
+
+      def check!(klass)
+        guard = @current
+        return unless guard && guard.mode != :off && guard.master?
+
+        locations = caller_locations(1, 8).reject { |location| location.path == __FILE__ }
+        violation = Violation.new(klass, locations)
+        guard.violations << violation
+        case guard.mode
+        when :raise then raise violation
+        when :warn
+          guard.logger ? guard.logger.warn(violation.message) : Kernel.warn(violation.message)
+        end
+      end
+    end
+
+    def initialize(mode:, logger:, master_pid:)
+      raise ArgumentError, "fork guard mode must be :raise, :warn, :off, or :record" unless %i[raise warn off record].include?(mode)
+
+      @mode = mode
+      @logger = logger
+      @master_pid = master_pid
+      @violations = []
+    end
+
+    def master?
+      Process.pid == @master_pid
+    end
+  end
+end

data/lib/gritz/metrics/aggregator.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Gritz
+  module Metrics
+    # Master-owned totals survive worker retirement; only replay state is forgotten.
+    # @api private
+    class Aggregator
+      def initialize
+        @lock = Mutex.new
+        @rpc = {}
+        @sequences = {}
+        @rejected = 0
+        @restarts = Hash.new(0)
+      end
+
+      def apply(worker, envelope)
+        unless envelope.is_a?(Hash) && envelope[:seq].is_a?(Integer) && envelope[:seq].positive?
+          raise ConfigurationError, "Invalid metric sequence"
+        end
+
+        @lock.synchronize do
+          sequence = envelope[:seq]
+          previous = @sequences.fetch(worker, 0)
+          return false if sequence <= previous
+          raise ConfigurationError, "Metric sequence gap: expected #{previous + 1}, got #{sequence}" unless sequence == previous + 1
+
+          delta = envelope[:delta]
+          validate_delta!(delta)
+          delta[:rpc].each do |row|
+            labels = row.slice(:service, :method, :code)
+            aggregate = @rpc[labels.values]
+            unless aggregate
+              aggregate = Recorder.empty_row(**labels)
+              @rpc[aggregate.values_at(:service, :method, :code).freeze] = aggregate
+            end
+            Recorder::SCALARS.each { |field| aggregate[field] += row[field] }
+            Recorder::BUCKET_FIELDS.each do |field|
+              row[field].each_with_index { |count, index| aggregate[field][index] += count }
+            end
+          end
+          @rejected += delta[:rejected]
+          @sequences[worker] = sequence
+          true
+        end
+      end
+
+      def forget(worker) = @lock.synchronize { @sequences.delete(worker) }
+
+      def record_restart(reason:)
+        @lock.synchronize { @restarts[reason.to_s] += 1 }
+      end
+
+      def render(workers: [])
+        rows = workers.map { |worker| worker.is_a?(Hash) ? worker : worker.stats.merge(pid: worker.pid, index: worker.index, state: worker.state) }
+        @lock.synchronize do
+          output = +""
+          histogram(output, "rpc_server_duration_seconds", Recorder::DURATION_BUCKETS, :duration_buckets, :duration_sum)
+          histogram(output, "rpc_server_requests_per_rpc", Recorder::MESSAGE_BUCKETS, :request_buckets, :request_sum)
+          histogram(output, "rpc_server_responses_per_rpc", Recorder::MESSAGE_BUCKETS, :response_buckets, :response_sum)
+          output << "# TYPE gritz_rejected_total counter\ngritz_rejected_total #{@rejected}\n"
+          output << "# TYPE gritz_worker_restarts_total counter\n"
+          @restarts.sort.each { |reason, count| output << "gritz_worker_restarts_total{reason=\"#{escape(reason)}\"} #{count}\n" }
+          output << "# TYPE gritz_workers gauge\n"
+          Supervisor::WorkerHandle::STATES.each { |state| output << "gritz_workers{state=\"#{state}\"} #{rows.count { |row| row[:state] == state }}\n" }
+          serving = rows.select { |row| %w[ready draining].include?(row[:state]) }
+          output << "# TYPE gritz_threadpool_busy gauge\ngritz_threadpool_busy #{serving.sum { |row| row[:busy_threads] || row[:busy] || 0 }}\n"
+          output << "# TYPE gritz_threadpool_capacity gauge\ngritz_threadpool_capacity #{serving.sum { |row| row[:capacity] || 0 }}\n"
+          output << "# TYPE gritz_worker_pss_bytes gauge\n"
+          rows.each do |row|
+            next unless row[:pss_bytes]
+
+            output << "gritz_worker_pss_bytes{worker=\"#{row[:index]}\",pid=\"#{row[:pid]}\"} #{row[:pss_bytes]}\n"
+          end
+          output
+        end
+      end
+
+      private
+
+      def validate_delta!(delta)
+        valid = delta.is_a?(Hash) && delta[:rpc].is_a?(Array) && nonnegative_integer?(delta[:rejected])
+        raise ConfigurationError, "Invalid metric delta" unless valid
+
+        delta[:rpc].each do |row|
+          valid = row.is_a?(Hash) && %i[service method].all? { |key| row[key].is_a?(String) && !row[key].empty? } &&
+                  row[:code].is_a?(Integer) && row[:code].between?(0, 16) &&
+                  %i[count request_sum response_sum].all? { |key| nonnegative_integer?(row[key]) } &&
+                  row[:duration_sum].is_a?(Numeric) && row[:duration_sum].real? && row[:duration_sum].to_f.finite? && row[:duration_sum] >= 0
+          raise ConfigurationError, "Invalid RPC metric row" unless valid
+
+          Recorder::BUCKET_FIELDS.each do |field|
+            size = field == :duration_buckets ? Recorder::DURATION_BUCKETS.size : Recorder::MESSAGE_BUCKETS.size
+            buckets = row[field]
+            unless buckets.is_a?(Array) && buckets.size == size && buckets.all? { |count| nonnegative_integer?(count) } && buckets.sum == row[:count]
+              raise ConfigurationError, "Invalid RPC metric histogram"
+            end
+          end
+        end
+      end
+
+      def nonnegative_integer?(value) = value.is_a?(Integer) && value >= 0
+
+      def histogram(output, name, bounds, bucket_field, sum_field)
+        output << "# TYPE #{name} histogram\n"
+        @rpc.sort_by { |key, _row| key }.each do |_key, row|
+          labels = "rpc_service=\"#{escape(row[:service])}\",rpc_method=\"#{escape(row[:method])}\",rpc_grpc_status_code=\"#{row[:code]}\""
+          total = 0
+          bounds.each_with_index do |upper, index|
+            total += row[bucket_field][index]
+            output << "#{name}_bucket{#{labels},le=\"#{upper.infinite? ? '+Inf' : upper}\"} #{total}\n"
+          end
+          output << "#{name}_sum{#{labels}} #{row[sum_field]}\n#{name}_count{#{labels}} #{row[:count]}\n"
+        end
+      end
+
+      def escape(value) = value.gsub(/[\\"\n]/) { |character| { "\\" => "\\\\", '"' => '\\"', "\n" => "\\n" }.fetch(character) }
+    end
+  end
+end