gritz-core 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 023ead58f761660de79d7168434d85be2f92e583073fab043b18b18fd14035c5
-  data.tar.gz: f319a9c5f6a190fad09e78667072ee22a5994dfe1f5c2e4f12287f3213865e1a
+  metadata.gz: 03bd5114b9267c0bc0b5daa80d14433d4941c753321338fc5b83b2281f4d6bf1
+  data.tar.gz: 67fabc1d94c5cca35149679a2e582c983a88122a5299638d05c639387ea48fa4
 SHA512:
-  metadata.gz: d531f928a935dd66d9a3459919fff84263309e6821da90332a841170b1bc268e2ccc711394a5873590611883136d44987d0d059794520d1335d788ad37815660
-  data.tar.gz: '08b50b33dcbb22fdde34c4052f2c15e82cfb1562bc83f450966f98879e4f0d8bc64cf50f8bb3517fae12ebfe4ac4841928f5ab5f5032a96eabd006711f8e44dc'
+  metadata.gz: 22ba419be6cd7bf1754eaccc7af3bde09ac0ca9dcac494f5d1932306aa73d595e0861b1c9ff1af9e694195f86d6c19e3103161a721ccd8e3f4c88ae72c93c56d
+  data.tar.gz: 1986b1cc2c4a32d0baf218b8268b1ced8399931c1a48379f46071f891c09d4c89cfe489fb707683dcf0d4ed239c708b3bfdd36a156efb0696f14d1764a10eeb6

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 0.3.0
+
+- Replace workers one at a time with `USR1` and recycle them by request count, memory or lifetime.
+- Reload application code with `USR2`; retain the serving master if replacement startup fails, and update the PID file after readiness.
+- Keep admin probes and Prometheus totals available across worker and master replacement. Add application health checks and per-worker status.
+- Emit one structured completion log per RPC with JSON or logfmt output and credential redaction.
+
 ## 0.2.0
 
 - Supervise forked workers with boot and heartbeat timeouts, automatic replacement, graceful shutdown, lifecycle hooks, dynamic worker counts and log reopening.

data/README.md

@@ -10,6 +10,8 @@ Use [gritz](https://github.com/gritzrpc/gritz) for the default framework and exe
 
 Controller tests can load `gritz/testing/rspec` or `gritz/testing/minitest` and run without a network or the grpc gem. See the [controller examples](https://github.com/gritzrpc/gritz#controllers) and [configuration guide](docs/guides/configuration.md).
 
+Supervised servers support phased replacement, fresh-interpreter hot reexec, request/memory/lifetime recycling, Admin HTTP probes and process-wide Prometheus totals. The CLI keeps one lifecycle owner across master generations; application controllers and gRPC resources stay in serving processes.
+
 ## Development
 
 ```sh

data/lib/gritz/cli.rb

@@ -2,16 +2,31 @@
 
 require "optparse"
 require "logger"
+require "rbconfig"
+require "socket"
 
 module Gritz
   # Starts a server, lists routes, or checks configuration and fork safety.
   # @api public
   class CLI
-    def initialize(stdout: $stdout, stderr: $stderr, env: ENV, status_io: nil)
+    CHILD_BOOTSTRAP = <<~RUBY
+      require "socket"
+      fd = ENV.delete("GRITZ_INTERNAL_OWNER_FD")
+      exit Gritz::CLI.new(owner_io: UNIXSocket.for_fd(Integer(fd))).run(ARGV)
+    RUBY
+
+    def self.child_command(argv)
+      [RbConfig.ruby, "-I", $LOAD_PATH.join(File::PATH_SEPARATOR), "-rgritz/core", "-e", CHILD_BOOTSTRAP, "--", *argv]
+    end
+
+    def initialize(stdout: $stdout, stderr: $stderr, env: ENV, status_io: nil, owner_io: nil, launch: false, launch_command: nil)
       @stdout = stdout
       @stderr = stderr
       @env = env
       @status_io = status_io
+      @owner_channel = Supervisor::StatusChannel.new(owner_io) if owner_io
+      @launch = launch
+      @launch_command = launch_command
     end
 
     def run(argv)
@@ -39,12 +54,17 @@ module Gritz
       raise ConfigurationError, "Unknown command #{command}" unless %w[start routes check].include?(command)
       raise ConfigurationError, "Unexpected argument #{args.first}" unless args.empty?
 
+      logger = Logger.new(@stdout)
+      logger.formatter = ->(_severity, _time, _progname, message) { "#{message}\n" }
+      if command == "start" && @launch && !@owner_channel
+        return Supervisor::Launcher.new(command: @launch_command || self.class.child_command(argv), env: @env,
+                                        stdout: @stdout, stderr: @stderr, logger: logger, status_io: @status_io).run
+      end
+
       path ||= "config/gritz.rb" if File.file?("config/gritz.rb")
       guard = ForkGuard.activate(mode: :record) if command != "routes"
       require "gritz/native" if command != "routes"
       config = Configuration.load(path: path, env: @env, overrides: overrides)
-      logger = Logger.new(@stdout)
-      logger.formatter = ->(_severity, _time, _progname, message) { "#{message}\n" }
       if command == "check"
         config.preload! if config.preload_app?
         guard.violations.each { |violation| @stderr.puts(violation.message) }
@@ -61,11 +81,16 @@ module Gritz
           guard.violations.each { |violation| logger.warn(violation.message) } if config.fork_guard == :warn
         end
         @stdout.sync = true if @stdout.respond_to?(:sync=)
-        return Supervisor::Master.new(config, logger: logger, status_io: @status_io).run if config.workers.positive?
+        @owner_channel&.write(type: "configured", pid: Process.pid, workers: config.workers, bind: config.bind,
+                              admin_bind: config.admin_bind, min_ready_workers: config.min_ready_workers, pid_file: config.pid_file,
+                              reexec_timeout: config.reexec_timeout, drain_delay: config.drain_delay, shutdown_timeout: config.shutdown_timeout)
+        if config.workers.positive?
+          return Supervisor::Master.new(config, logger: logger, status_io: @status_io, owner_channel: @owner_channel).run
+        end
 
         ForkGuard.deactivate
         config.preload! if config.preload_app?
-        return Worker::Runner.new(index: 0, config: config, logger: logger).run
+        return Worker::Runner.new(index: 0, config: config, logger: logger, owner_channel: @owner_channel).run
       else
         config.preload! if config.preload_app?
         router = Router.new(controllers: config.controllers, strict: config.strict_routes, logger: logger)
@@ -81,6 +106,7 @@ module Gritz
       1
     ensure
       ForkGuard.deactivate if guard
+      @owner_channel&.close
     end
   end
 end

data/lib/gritz/configuration.rb

@@ -18,7 +18,8 @@ module Gritz
       keepalive_permit_without_calls: true,
       max_receive_message_size: 4 * 1024 * 1024,
       max_send_message_size: 4 * 1024 * 1024, max_metadata_size: 8192,
-      metrics_backend: :pipe, log_format: :json, worker_recycle: {}, tls: {}
+      metrics_backend: :pipe, log_format: :json, log_redact: [], worker_recycle: {}, tls: {},
+      pid_file: "", reexec_timeout: 60.0
     }.freeze
     ENUMS = {
       transport: %i[native async], listener_strategy: %i[reuseport inherited_fd port_per_worker],
@@ -63,7 +64,7 @@ module Gritz
       when Integer then Integer(value, 10)
       when Float then Float(value)
       when Symbol then value.to_sym
-      when Hash then JSON.parse(value, symbolize_names: true)
+      when Hash, Array then JSON.parse(value, symbolize_names: true)
       when true, false
         return true if %w[true 1].include?(value)
         return false if %w[false 0].include?(value)
@@ -86,7 +87,7 @@ module Gritz
                 end
         raise ConfigurationError, "Invalid #{name}: #{value.inspect}" unless valid
       end
-      %i[shutdown_timeout worker_boot_timeout worker_timeout status_interval].each do |name|
+      %i[shutdown_timeout worker_boot_timeout worker_timeout status_interval reexec_timeout].each do |name|
         raise ConfigurationError, "#{name} must be positive" unless public_send(name).positive?
       end
       ENUMS.each do |name, allowed|
@@ -98,6 +99,12 @@ module Gritz
       raise ConfigurationError, "controllers must be an Array of classes" unless controllers.is_a?(Array) && controllers.all?(Class)
       raise ConfigurationError, "middleware must be a Stack" unless middleware.is_a?(Middleware::Stack)
       raise ConfigurationError, "preload_app must be boolean" unless [true, false].include?(preload_app)
+      unless log_redact.all? { |name| name.is_a?(String) || name.is_a?(Symbol) }
+        raise ConfigurationError, "log_redact must contain metadata names"
+      end
+      unless health_checks.all? { |name, check| (name.is_a?(String) || name.is_a?(Symbol)) && check.respond_to?(:call) }
+        raise ConfigurationError, "health_checks must contain named callbacks"
+      end
 
       self
     end
@@ -109,11 +116,13 @@ module Gritz
       validate!
       raise ConfigurationError, "This release supports transport :native" unless transport == :native
       raise ConfigurationError, "This release supports listener_strategy :reuseport" unless listener_strategy == :reuseport
-      raise ConfigurationError, "TLS is planned for v0.3" unless tls.empty?
-      raise ConfigurationError, "worker_recycle is planned for v0.3" unless worker_recycle.empty?
-      raise ConfigurationError, "Health checks are planned for v0.3" unless health_checks.empty?
-      raise ConfigurationError, "This release supports log_format :json" unless log_format == :json
-      raise ConfigurationError, "This release reserves metrics_backend :pipe; metrics export is planned for v0.3" unless metrics_backend == :pipe
+      raise ConfigurationError, "This release supports metrics_backend :pipe" unless metrics_backend == :pipe
+      raise ConfigurationError, "This release supports phased_restart_surge 1" unless phased_restart_surge == 1
+
+      unless worker_recycle.empty?
+        raise ConfigurationError, "worker_recycle requires workers > 0" unless workers.positive?
+        raise ConfigurationError, "worker_recycle requires a fixed bind port" if bind.end_with?(":0")
+      end
       raise ConfigurationError, "grpc_fork_support requires workers > 0" if workers.zero? && fork_mode != :clean
       raise ConfigurationError, "at least one controller must be registered" if controllers.empty?
 

data/lib/gritz/context.rb

@@ -4,13 +4,19 @@ module Gritz
   # Request state inherited by child fibers and threads using Ruby Fiber storage.
   # @api public
   class Context
-    attr_reader :call, :logger, :store
+    attr_reader :call, :logger, :store, :metrics, :worker, :log_format, :log_redact,
+                :requests_count, :responses_count, :bytes_in, :bytes_out
     attr_accessor :request_id, :trace_context
 
-    def initialize(call:, logger:)
+    def initialize(call:, logger:, metrics: nil, worker: nil, log_format: :json, log_redact: [])
       @call = call
       @logger = logger
       @store = {}
+      @metrics = metrics
+      @worker = worker
+      @log_format = log_format
+      @log_redact = log_redact.map(&:to_s)
+      @requests_count = @responses_count = @bytes_in = @bytes_out = 0
     end
 
     def self.current = Fiber[:gritz_context]
@@ -46,5 +52,28 @@ module Gritz
 
       message
     end
+
+    # @api private
+    def record_received(message)
+      return unless message
+
+      size = encoded_size(message)
+      @requests_count += 1
+      @bytes_in += size
+    end
+
+    # @api private
+    def record_sent(message)
+      size = encoded_size(message)
+      @responses_count += 1
+      @bytes_out += size
+    end
+
+    private
+
+    def encoded_size(message)
+      # ponytail: byte accounting encodes again; instrument native serialization if the Phase 6 overhead budget is exceeded.
+      message.class.respond_to?(:encode) ? message.class.encode(message).bytesize : message.to_s.bytesize
+    end
   end
 end

data/lib/gritz/controller.rb

@@ -96,6 +96,10 @@ module Gritz
       def initialize(call, context)
         @call = call
         @context = context
+        unless context.method.client_streaming?
+          @message = @call.read
+          @context.record_received(@message)
+        end
       end
 
       def message
@@ -104,14 +108,23 @@ module Gritz
         return @message if defined?(@message)
 
         @message = @call.read
+        @context.record_received(@message)
+        @message
       end
 
       def each_message
         return enum_for(__method__) unless block_given?
 
+        unless @context.method.client_streaming?
+          value = message
+          yield value if value
+          return
+        end
+
         @call.each_message do |message|
           @context.check_deadline!
           @context.check_cancelled!
+          @context.record_received(message)
           yield message
         end
       end
@@ -133,6 +146,7 @@ module Gritz
         @context.check_cancelled!
         @context.validate_response!(message)
         @call.write(message)
+        @context.record_sent(message)
       end
     end
   end

data/lib/gritz/core/version.rb

@@ -2,6 +2,6 @@
 
 module Gritz
   module Core
-    VERSION = "0.2.0"
+    VERSION = "0.3.0"
   end
 end

data/lib/gritz/dispatcher.rb

@@ -8,9 +8,11 @@ module Gritz
   class Dispatcher
     attr_reader :router
 
-    def initialize(router:, middleware: Middleware::Stack.default, logger: Logger.new($stdout))
+    def initialize(router:, middleware: Middleware::Stack.default, logger: Logger.new($stdout),
+                   metrics: nil, worker: nil, log_format: :json, log_redact: [])
       @router = router
       @logger = logger
+      @context_options = { metrics: metrics, worker: worker, log_format: log_format, log_redact: log_redact }
       endpoint = lambda do |context|
         context.check_deadline!
         context.check_cancelled!
@@ -18,13 +20,17 @@ module Gritz
         result = descriptor.controller.new(context:).process(descriptor.action)
         context.check_deadline!
         context.check_cancelled!
-        descriptor.server_streaming? ? result : context.validate_response!(result)
+        unless descriptor.server_streaming?
+          context.validate_response!(result)
+          context.record_sent(result)
+        end
+        result
       end
       @app = middleware ? middleware.build(endpoint) : endpoint
     end
 
     def call(call)
-      @app.call(Context.new(call:, logger: @logger))
+      @app.call(Context.new(call:, logger: @logger, **@context_options))
     end
   end
 end

data/lib/gritz/metrics/aggregator.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Gritz
+  module Metrics
+    # Master-owned totals survive worker retirement; only replay state is forgotten.
+    # @api private
+    class Aggregator
+      def initialize
+        @lock = Mutex.new
+        @rpc = {}
+        @sequences = {}
+        @rejected = 0
+        @restarts = Hash.new(0)
+      end
+
+      def apply(worker, envelope)
+        unless envelope.is_a?(Hash) && envelope[:seq].is_a?(Integer) && envelope[:seq].positive?
+          raise ConfigurationError, "Invalid metric sequence"
+        end
+
+        @lock.synchronize do
+          sequence = envelope[:seq]
+          previous = @sequences.fetch(worker, 0)
+          return false if sequence <= previous
+          raise ConfigurationError, "Metric sequence gap: expected #{previous + 1}, got #{sequence}" unless sequence == previous + 1
+
+          delta = envelope[:delta]
+          validate_delta!(delta)
+          delta[:rpc].each do |row|
+            labels = row.slice(:service, :method, :code)
+            aggregate = @rpc[labels.values]
+            unless aggregate
+              aggregate = Recorder.empty_row(**labels)
+              @rpc[aggregate.values_at(:service, :method, :code).freeze] = aggregate
+            end
+            Recorder::SCALARS.each { |field| aggregate[field] += row[field] }
+            Recorder::BUCKET_FIELDS.each do |field|
+              row[field].each_with_index { |count, index| aggregate[field][index] += count }
+            end
+          end
+          @rejected += delta[:rejected]
+          @sequences[worker] = sequence
+          true
+        end
+      end
+
+      def forget(worker) = @lock.synchronize { @sequences.delete(worker) }
+
+      def record_restart(reason:)
+        @lock.synchronize { @restarts[reason.to_s] += 1 }
+      end
+
+      def render(workers: [])
+        rows = workers.map { |worker| worker.is_a?(Hash) ? worker : worker.stats.merge(pid: worker.pid, index: worker.index, state: worker.state) }
+        @lock.synchronize do
+          output = +""
+          histogram(output, "rpc_server_duration_seconds", Recorder::DURATION_BUCKETS, :duration_buckets, :duration_sum)
+          histogram(output, "rpc_server_requests_per_rpc", Recorder::MESSAGE_BUCKETS, :request_buckets, :request_sum)
+          histogram(output, "rpc_server_responses_per_rpc", Recorder::MESSAGE_BUCKETS, :response_buckets, :response_sum)
+          output << "# TYPE gritz_rejected_total counter\ngritz_rejected_total #{@rejected}\n"
+          output << "# TYPE gritz_worker_restarts_total counter\n"
+          @restarts.sort.each { |reason, count| output << "gritz_worker_restarts_total{reason=\"#{escape(reason)}\"} #{count}\n" }
+          output << "# TYPE gritz_workers gauge\n"
+          Supervisor::WorkerHandle::STATES.each { |state| output << "gritz_workers{state=\"#{state}\"} #{rows.count { |row| row[:state] == state }}\n" }
+          serving = rows.select { |row| %w[ready draining].include?(row[:state]) }
+          output << "# TYPE gritz_threadpool_busy gauge\ngritz_threadpool_busy #{serving.sum { |row| row[:busy_threads] || row[:busy] || 0 }}\n"
+          output << "# TYPE gritz_threadpool_capacity gauge\ngritz_threadpool_capacity #{serving.sum { |row| row[:capacity] || 0 }}\n"
+          output << "# TYPE gritz_worker_pss_bytes gauge\n"
+          rows.each do |row|
+            next unless row[:pss_bytes]
+
+            output << "gritz_worker_pss_bytes{worker=\"#{row[:index]}\",pid=\"#{row[:pid]}\"} #{row[:pss_bytes]}\n"
+          end
+          output
+        end
+      end
+
+      private
+
+      def validate_delta!(delta)
+        valid = delta.is_a?(Hash) && delta[:rpc].is_a?(Array) && nonnegative_integer?(delta[:rejected])
+        raise ConfigurationError, "Invalid metric delta" unless valid
+
+        delta[:rpc].each do |row|
+          valid = row.is_a?(Hash) && %i[service method].all? { |key| row[key].is_a?(String) && !row[key].empty? } &&
+                  row[:code].is_a?(Integer) && row[:code].between?(0, 16) &&
+                  %i[count request_sum response_sum].all? { |key| nonnegative_integer?(row[key]) } &&
+                  row[:duration_sum].is_a?(Numeric) && row[:duration_sum].real? && row[:duration_sum].to_f.finite? && row[:duration_sum] >= 0
+          raise ConfigurationError, "Invalid RPC metric row" unless valid
+
+          Recorder::BUCKET_FIELDS.each do |field|
+            size = field == :duration_buckets ? Recorder::DURATION_BUCKETS.size : Recorder::MESSAGE_BUCKETS.size
+            buckets = row[field]
+            unless buckets.is_a?(Array) && buckets.size == size && buckets.all? { |count| nonnegative_integer?(count) } && buckets.sum == row[:count]
+              raise ConfigurationError, "Invalid RPC metric histogram"
+            end
+          end
+        end
+      end
+
+      def nonnegative_integer?(value) = value.is_a?(Integer) && value >= 0
+
+      def histogram(output, name, bounds, bucket_field, sum_field)
+        output << "# TYPE #{name} histogram\n"
+        @rpc.sort_by { |key, _row| key }.each do |_key, row|
+          labels = "rpc_service=\"#{escape(row[:service])}\",rpc_method=\"#{escape(row[:method])}\",rpc_grpc_status_code=\"#{row[:code]}\""
+          total = 0
+          bounds.each_with_index do |upper, index|
+            total += row[bucket_field][index]
+            output << "#{name}_bucket{#{labels},le=\"#{upper.infinite? ? '+Inf' : upper}\"} #{total}\n"
+          end
+          output << "#{name}_sum{#{labels}} #{row[sum_field]}\n#{name}_count{#{labels}} #{row[:count]}\n"
+        end
+      end
+
+      def escape(value) = value.gsub(/[\\"\n]/) { |character| { "\\" => "\\\\", '"' => '\\"', "\n" => "\\n" }.fetch(character) }
+    end
+  end
+end

data/lib/gritz/metrics/recorder.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Gritz
+  module Metrics
+    # Per-worker RPC deltas; only one bounded detached batch is needed by the sender.
+    # @api private
+    class Recorder
+      DURATION_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, Float::INFINITY].freeze
+      MESSAGE_BUCKETS = [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 1024, Float::INFINITY].freeze
+      SCALARS = %i[count duration_sum request_sum response_sum].freeze
+      BUCKET_FIELDS = %i[duration_buckets request_buckets response_buckets].freeze
+
+      def initialize
+        @lock = Mutex.new
+        @rpc = {}
+        @last_rejected = @rejected_delta = 0
+      end
+
+      def self.empty_row(service:, method:, code:)
+        { service: service.dup.freeze, method: method.dup.freeze, code: code, count: 0, duration_sum: 0.0, request_sum: 0, response_sum: 0,
+          duration_buckets: Array.new(DURATION_BUCKETS.size, 0), request_buckets: Array.new(MESSAGE_BUCKETS.size, 0),
+          response_buckets: Array.new(MESSAGE_BUCKETS.size, 0) }
+      end
+
+      def record_rpc(service:, method:, code:, duration:, requests:, responses:)
+        unless [service, method].all? { |label| label.is_a?(String) && !label.empty? } && code.is_a?(Integer) && code.between?(0, 16) &&
+               duration.is_a?(Numeric) && duration.real? && duration.to_f.finite? && duration >= 0 &&
+               [requests, responses].all? { |value| value.is_a?(Integer) && value >= 0 }
+          raise ArgumentError, "Invalid RPC metric observation"
+        end
+
+        @lock.synchronize do
+          row = @rpc[[service, method, code]]
+          unless row
+            row = self.class.empty_row(service: service, method: method, code: code)
+            @rpc[[row[:service], row[:method], code].freeze] = row
+          end
+          row[:count] += 1
+          row[:duration_sum] += duration.to_f
+          row[:request_sum] += requests
+          row[:response_sum] += responses
+          row[:duration_buckets][DURATION_BUCKETS.bsearch_index { |upper| duration <= upper }] += 1
+          row[:request_buckets][MESSAGE_BUCKETS.bsearch_index { |upper| requests <= upper }] += 1
+          row[:response_buckets][MESSAGE_BUCKETS.bsearch_index { |upper| responses <= upper }] += 1
+        end
+      end
+
+      def observe_rejected(total)
+        @lock.synchronize do
+          raise ArgumentError, "Rejected counter must be a nondecreasing integer" unless total.is_a?(Integer) && total >= @last_rejected
+
+          @rejected_delta += total - @last_rejected
+          @last_rejected = total
+        end
+      end
+
+      # The caller retains this detached delta until its status channel accepts the row.
+      def take_delta(max_bytes: Supervisor::StatusChannel::MAX_LINE_BYTES - 4096)
+        raise ArgumentError, "Metric packet budget must be positive" unless max_bytes.is_a?(Integer) && max_bytes.positive?
+
+        @lock.synchronize do
+          return nil if @rpc.empty? && @rejected_delta.zero?
+
+          delta = { rpc: [], rejected: @rejected_delta }
+          used_bytes = JSON.generate(delta).bytesize
+          raise ArgumentError, "Metric packet budget is too small" if used_bytes > max_bytes
+
+          selected = []
+          @rpc.each do |key, row|
+            row_bytes = JSON.generate(row).bytesize + (selected.empty? ? 0 : 1)
+            if used_bytes + row_bytes > max_bytes
+              raise ArgumentError, "Metric row exceeds the packet budget" if selected.empty?
+
+              break
+            end
+            delta[:rpc] << row
+            selected << key
+            used_bytes += row_bytes
+          end
+          selected.each { |key| @rpc.delete(key) }
+          @rejected_delta = 0
+          delta
+        end
+      end
+    end
+  end
+end

data/lib/gritz/middleware/exception_mapper.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "securerandom"
-require "json"
 
 module Gritz
   module Middleware
@@ -32,8 +31,7 @@ module Gritz
         end
 
         error_id = SecureRandom.uuid
-        context.logger.error(JSON.generate(error_id:, request_id: context.request_id, error: e.class.name,
-                                           message: e.message, backtrace: e.backtrace))
+        context.store[:gritz_error] = { error_id: error_id, error: e.class.name, message: e.message, backtrace: e.backtrace }
         raise Errors::Internal.new(@expose_errors ? e.message : "internal error (#{error_id})", metadata: { "error-id" => error_id })
       end
     end

data/lib/gritz/middleware/logging.rb

@@ -23,9 +23,33 @@ module Gritz
         raise
       ensure
         duration = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000
-        context.logger.info(JSON.generate(request_id: context.request_id, service: context.method.service,
-                                          method: context.method.name, code:, duration_ms: duration.round(3),
-                                          peer: context.peer, pid: Process.pid))
+        fields = { request_id: context.request_id, service: context.method.service,
+                   method: context.method.name, code: code, duration_ms: duration.round(3),
+                   peer: context.peer, worker: context.worker, pid: Process.pid, bytes_in: context.bytes_in, bytes_out: context.bytes_out }
+        fields.merge!(context.store.fetch(:gritz_error, {}))
+        fields = prepare(fields, context.log_redact)
+        message = if context.log_format == :logfmt
+                    fields.map do |key, value|
+                      value = value.to_s if value.is_a?(Symbol)
+                      value = JSON.generate(value) if value.is_a?(Hash) || value.is_a?(Array)
+                      "#{key}=#{JSON.generate(value)}"
+                    end.join(" ")
+                  else
+                    JSON.generate(fields)
+                  end
+        context.logger.info(message)
+      end
+
+      private
+
+      def prepare(value, redacted)
+        case value
+        when Hash
+          value.to_h { |key, item| [key, redacted.include?(key.to_s) ? "[FILTERED]" : prepare(item, redacted)] }
+        when Array then value.map { |item| prepare(item, redacted) }
+        when String then value.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+        else value
+        end
       end
     end
   end

data/lib/gritz/middleware/metrics.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gritz
+  module Middleware
+    # One completion observation wraps the complete unary or streaming action.
+    # @api public
+    class Metrics
+      def initialize(app)
+        @app = app
+      end
+
+      def call(context)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        code = 0
+        @app.call(context)
+      rescue Gritz::Error => e
+        code = e.grpc_code
+        raise
+      rescue StandardError
+        code = 13
+        raise
+      ensure
+        context.metrics&.record_rpc(service: context.method.service, method: context.method.name, code: code,
+                                    duration: Process.clock_gettime(Process::CLOCK_MONOTONIC) - started,
+                                    requests: context.requests_count, responses: context.responses_count)
+      end
+    end
+  end
+end

data/lib/gritz/middleware/stack.rb

@@ -18,7 +18,7 @@ module Gritz
       end
 
       def self.default
-        new.use(RequestId).use(Context).use(Logging).use(ExceptionMapper)
+        new.use(RequestId).use(Context).use(Metrics).use(Logging).use(ExceptionMapper)
       end
 
       def use(middleware, **options, &block)