gritz-core 0.2.0 → 0.4.0

data/lib/gritz/metrics/aggregator.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Gritz
+  module Metrics
+    # Master-owned totals survive worker retirement; only replay state is forgotten.
+    # @api private
+    class Aggregator
+      def initialize
+        @lock = Mutex.new
+        @rpc = {}
+        @sequences = {}
+        @rejected = 0
+        @restarts = Hash.new(0)
+      end
+
+      def apply(worker, envelope)
+        unless envelope.is_a?(Hash) && envelope[:seq].is_a?(Integer) && envelope[:seq].positive?
+          raise ConfigurationError, "Invalid metric sequence"
+        end
+
+        @lock.synchronize do
+          sequence = envelope[:seq]
+          previous = @sequences.fetch(worker, 0)
+          return false if sequence <= previous
+          raise ConfigurationError, "Metric sequence gap: expected #{previous + 1}, got #{sequence}" unless sequence == previous + 1
+
+          delta = envelope[:delta]
+          validate_delta!(delta)
+          delta[:rpc].each do |row|
+            labels = row.slice(:service, :method, :code)
+            aggregate = @rpc[labels.values]
+            unless aggregate
+              aggregate = Recorder.empty_row(**labels)
+              @rpc[aggregate.values_at(:service, :method, :code).freeze] = aggregate
+            end
+            Recorder::SCALARS.each { |field| aggregate[field] += row[field] }
+            Recorder::BUCKET_FIELDS.each do |field|
+              row[field].each_with_index { |count, index| aggregate[field][index] += count }
+            end
+          end
+          @rejected += delta[:rejected]
+          @sequences[worker] = sequence
+          true
+        end
+      end
+
+      def forget(worker) = @lock.synchronize { @sequences.delete(worker) }
+
+      def record_restart(reason:)
+        @lock.synchronize { @restarts[reason.to_s] += 1 }
+      end
+
+      def render(workers: [])
+        rows = workers.map { |worker| worker.is_a?(Hash) ? worker : worker.stats.merge(pid: worker.pid, index: worker.index, state: worker.state) }
+        @lock.synchronize do
+          output = +""
+          histogram(output, "rpc_server_duration_seconds", Recorder::DURATION_BUCKETS, :duration_buckets, :duration_sum)
+          histogram(output, "rpc_server_requests_per_rpc", Recorder::MESSAGE_BUCKETS, :request_buckets, :request_sum)
+          histogram(output, "rpc_server_responses_per_rpc", Recorder::MESSAGE_BUCKETS, :response_buckets, :response_sum)
+          output << "# TYPE gritz_rejected_total counter\ngritz_rejected_total #{@rejected}\n"
+          output << "# TYPE gritz_worker_restarts_total counter\n"
+          @restarts.sort.each { |reason, count| output << "gritz_worker_restarts_total{reason=\"#{escape(reason)}\"} #{count}\n" }
+          output << "# TYPE gritz_workers gauge\n"
+          Supervisor::WorkerHandle::STATES.each { |state| output << "gritz_workers{state=\"#{state}\"} #{rows.count { |row| row[:state] == state }}\n" }
+          serving = rows.select { |row| %w[ready draining].include?(row[:state]) }
+          output << "# TYPE gritz_threadpool_busy gauge\ngritz_threadpool_busy #{serving.sum { |row| row[:busy_threads] || row[:busy] || 0 }}\n"
+          output << "# TYPE gritz_threadpool_capacity gauge\ngritz_threadpool_capacity #{serving.sum { |row| row[:capacity] || 0 }}\n"
+          output << "# TYPE gritz_worker_pss_bytes gauge\n"
+          rows.each do |row|
+            next unless row[:pss_bytes]
+
+            output << "gritz_worker_pss_bytes{worker=\"#{row[:index]}\",pid=\"#{row[:pid]}\"} #{row[:pss_bytes]}\n"
+          end
+          output
+        end
+      end
+
+      private
+
+      def validate_delta!(delta)
+        valid = delta.is_a?(Hash) && delta[:rpc].is_a?(Array) && nonnegative_integer?(delta[:rejected])
+        raise ConfigurationError, "Invalid metric delta" unless valid
+
+        delta[:rpc].each do |row|
+          valid = row.is_a?(Hash) && %i[service method].all? { |key| row[key].is_a?(String) && !row[key].empty? } &&
+                  row[:code].is_a?(Integer) && row[:code].between?(0, 16) &&
+                  %i[count request_sum response_sum].all? { |key| nonnegative_integer?(row[key]) } &&
+                  row[:duration_sum].is_a?(Numeric) && row[:duration_sum].real? && row[:duration_sum].to_f.finite? && row[:duration_sum] >= 0
+          raise ConfigurationError, "Invalid RPC metric row" unless valid
+
+          Recorder::BUCKET_FIELDS.each do |field|
+            size = field == :duration_buckets ? Recorder::DURATION_BUCKETS.size : Recorder::MESSAGE_BUCKETS.size
+            buckets = row[field]
+            unless buckets.is_a?(Array) && buckets.size == size && buckets.all? { |count| nonnegative_integer?(count) } && buckets.sum == row[:count]
+              raise ConfigurationError, "Invalid RPC metric histogram"
+            end
+          end
+        end
+      end
+
+      def nonnegative_integer?(value) = value.is_a?(Integer) && value >= 0
+
+      def histogram(output, name, bounds, bucket_field, sum_field)
+        output << "# TYPE #{name} histogram\n"
+        @rpc.sort_by { |key, _row| key }.each do |_key, row|
+          labels = "rpc_service=\"#{escape(row[:service])}\",rpc_method=\"#{escape(row[:method])}\",rpc_grpc_status_code=\"#{row[:code]}\""
+          total = 0
+          bounds.each_with_index do |upper, index|
+            total += row[bucket_field][index]
+            output << "#{name}_bucket{#{labels},le=\"#{upper.infinite? ? '+Inf' : upper}\"} #{total}\n"
+          end
+          output << "#{name}_sum{#{labels}} #{row[sum_field]}\n#{name}_count{#{labels}} #{row[:count]}\n"
+        end
+      end
+
+      def escape(value) = value.gsub(/[\\"\n]/) { |character| { "\\" => "\\\\", '"' => '\\"', "\n" => "\\n" }.fetch(character) }
+    end
+  end
+end

data/lib/gritz/metrics/recorder.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Gritz
+  module Metrics
+    # Per-worker RPC deltas; only one bounded detached batch is needed by the sender.
+    # @api private
+    class Recorder
+      DURATION_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, Float::INFINITY].freeze
+      MESSAGE_BUCKETS = [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 1024, Float::INFINITY].freeze
+      SCALARS = %i[count duration_sum request_sum response_sum].freeze
+      BUCKET_FIELDS = %i[duration_buckets request_buckets response_buckets].freeze
+
+      def initialize
+        @lock = Mutex.new
+        @rpc = {}
+        @last_rejected = @rejected_delta = 0
+      end
+
+      def self.empty_row(service:, method:, code:)
+        { service: service.dup.freeze, method: method.dup.freeze, code: code, count: 0, duration_sum: 0.0, request_sum: 0, response_sum: 0,
+          duration_buckets: Array.new(DURATION_BUCKETS.size, 0), request_buckets: Array.new(MESSAGE_BUCKETS.size, 0),
+          response_buckets: Array.new(MESSAGE_BUCKETS.size, 0) }
+      end
+
+      def record_rpc(service:, method:, code:, duration:, requests:, responses:)
+        unless [service, method].all? { |label| label.is_a?(String) && !label.empty? } && code.is_a?(Integer) && code.between?(0, 16) &&
+               duration.is_a?(Numeric) && duration.real? && duration.to_f.finite? && duration >= 0 &&
+               [requests, responses].all? { |value| value.is_a?(Integer) && value >= 0 }
+          raise ArgumentError, "Invalid RPC metric observation"
+        end
+
+        @lock.synchronize do
+          row = @rpc[[service, method, code]]
+          unless row
+            row = self.class.empty_row(service: service, method: method, code: code)
+            @rpc[[row[:service], row[:method], code].freeze] = row
+          end
+          row[:count] += 1
+          row[:duration_sum] += duration.to_f
+          row[:request_sum] += requests
+          row[:response_sum] += responses
+          row[:duration_buckets][DURATION_BUCKETS.bsearch_index { |upper| duration <= upper }] += 1
+          row[:request_buckets][MESSAGE_BUCKETS.bsearch_index { |upper| requests <= upper }] += 1
+          row[:response_buckets][MESSAGE_BUCKETS.bsearch_index { |upper| responses <= upper }] += 1
+        end
+      end
+
+      def observe_rejected(total)
+        @lock.synchronize do
+          raise ArgumentError, "Rejected counter must be a nondecreasing integer" unless total.is_a?(Integer) && total >= @last_rejected
+
+          @rejected_delta += total - @last_rejected
+          @last_rejected = total
+        end
+      end
+
+      # Optional process observations and shutdown for installed worker backends.
+      def observe_worker(_status); end
+      def close(timeout: nil); end
+
+      # The caller retains this detached delta until its status channel accepts the row.
+      def take_delta(max_bytes: Supervisor::StatusChannel::MAX_LINE_BYTES - 4096)
+        raise ArgumentError, "Metric packet budget must be positive" unless max_bytes.is_a?(Integer) && max_bytes.positive?
+
+        @lock.synchronize do
+          return nil if @rpc.empty? && @rejected_delta.zero?
+
+          delta = { rpc: [], rejected: @rejected_delta }
+          used_bytes = JSON.generate(delta).bytesize
+          raise ArgumentError, "Metric packet budget is too small" if used_bytes > max_bytes
+
+          selected = []
+          @rpc.each do |key, row|
+            row_bytes = JSON.generate(row).bytesize + (selected.empty? ? 0 : 1)
+            if used_bytes + row_bytes > max_bytes
+              raise ArgumentError, "Metric row exceeds the packet budget" if selected.empty?
+
+              break
+            end
+            delta[:rpc] << row
+            selected << key
+            used_bytes += row_bytes
+          end
+          selected.each { |key| @rpc.delete(key) }
+          @rejected_delta = 0
+          delta
+        end
+      end
+    end
+  end
+end

data/lib/gritz/middleware/exception_mapper.rb

@@ -1,23 +1,25 @@
 # frozen_string_literal: true
 
 require "securerandom"
-require "json"
 
 module Gritz
   module Middleware
     # Converts application exceptions to safe RPC errors.
     # @api public
     class ExceptionMapper
-      def initialize(app, mappings: {}, expose_errors: false)
+      def initialize(app, mappings: {}, expose_errors: false, passthrough_remote_errors: false)
         @app = app
         @mappings = mappings
         @expose_errors = expose_errors
+        @passthrough_remote_errors = passthrough_remote_errors
       end
 
       def call(context)
         @app.call(context)
-      rescue Gritz::Error
-        raise
+      rescue Gritz::Error => e
+        raise unless e.remote? && !@passthrough_remote_errors
+
+        raise_internal(context, e)
       rescue StandardError => e
         mapping = @mappings.find { |klass, _| e.is_a?(klass) }&.last
         if mapping
@@ -31,10 +33,15 @@ module Gritz
           raise Errors::InvalidArgument, "invalid record"
         end
 
+        raise_internal(context, e)
+      end
+
+      private
+
+      def raise_internal(context, error)
         error_id = SecureRandom.uuid
-        context.logger.error(JSON.generate(error_id:, request_id: context.request_id, error: e.class.name,
-                                           message: e.message, backtrace: e.backtrace))
-        raise Errors::Internal.new(@expose_errors ? e.message : "internal error (#{error_id})", metadata: { "error-id" => error_id })
+        context.store[:gritz_error] = { error_id: error_id, error: error.class.name, message: error.message, backtrace: error.backtrace }
+        raise Errors::Internal.new(@expose_errors ? error.message : "internal error (#{error_id})", metadata: { "error-id" => error_id })
       end
     end
   end

data/lib/gritz/middleware/logging.rb

@@ -23,9 +23,33 @@ module Gritz
         raise
       ensure
         duration = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000
-        context.logger.info(JSON.generate(request_id: context.request_id, service: context.method.service,
-                                          method: context.method.name, code:, duration_ms: duration.round(3),
-                                          peer: context.peer, pid: Process.pid))
+        fields = { request_id: context.request_id, service: context.method.service,
+                   method: context.method.name, code: code, duration_ms: duration.round(3),
+                   peer: context.peer, worker: context.worker, pid: Process.pid, bytes_in: context.bytes_in, bytes_out: context.bytes_out }
+        fields.merge!(context.store.fetch(:gritz_error, {}))
+        fields = prepare(fields, context.log_redact)
+        message = if context.log_format == :logfmt
+                    fields.map do |key, value|
+                      value = value.to_s if value.is_a?(Symbol)
+                      value = JSON.generate(value) if value.is_a?(Hash) || value.is_a?(Array)
+                      "#{key}=#{JSON.generate(value)}"
+                    end.join(" ")
+                  else
+                    JSON.generate(fields)
+                  end
+        context.logger.info(message)
+      end
+
+      private
+
+      def prepare(value, redacted)
+        case value
+        when Hash
+          value.to_h { |key, item| [key, redacted.include?(key.to_s) ? "[FILTERED]" : prepare(item, redacted)] }
+        when Array then value.map { |item| prepare(item, redacted) }
+        when String then value.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+        else value
+        end
       end
     end
   end

data/lib/gritz/middleware/metrics.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gritz
+  module Middleware
+    # One completion observation wraps the complete unary or streaming action.
+    # @api public
+    class Metrics
+      def initialize(app)
+        @app = app
+      end
+
+      def call(context)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        code = 0
+        @app.call(context)
+      rescue Gritz::Error => e
+        code = e.grpc_code
+        raise
+      rescue StandardError
+        code = 13
+        raise
+      ensure
+        context.metrics&.record_rpc(service: context.method.service, method: context.method.name, code: code,
+                                    duration: Process.clock_gettime(Process::CLOCK_MONOTONIC) - started,
+                                    requests: context.requests_count, responses: context.responses_count)
+      end
+    end
+  end
+end

data/lib/gritz/middleware/stack.rb

@@ -18,7 +18,7 @@ module Gritz
       end
 
       def self.default
-        new.use(RequestId).use(Context).use(Logging).use(ExceptionMapper)
+        new.use(RequestId).use(Context).use(Metrics).use(Logging).use(ExceptionMapper)
       end
 
       def use(middleware, **options, &block)

data/lib/gritz/supervisor/admin_server.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require "socket"
+require "json"
+
+module Gritz
+  module Supervisor
+    # Small, bounded HTTP probe listener polled by its owning process.
+    # @api private
+    class AdminServer
+      MAX_CLIENTS = 32
+      MAX_HEADER_BYTES = 8192
+      CLIENT_TIMEOUT = 2.0
+      attr_reader :io, :address
+
+      def initialize(bind:, status:, ready:, metrics:, logger:)
+        host, port = bind.match(/\A(\[[^\]]+\]|[^:]+):(\d+)\z/).captures
+        host = host.delete_prefix("[").delete_suffix("]")
+        @io = TCPServer.new(host, Integer(port))
+        @address = "#{host.include?(':') ? "[#{host}]" : host}:#{@io.addr[1]}"
+        @status = status
+        @ready = ready
+        @metrics = metrics
+        @logger = logger
+        @clients = {}
+      end
+
+      def ios = [@io, *@clients.keys]
+
+      def poll
+        4.times do
+          socket = @io.accept_nonblock(exception: false)
+          break if socket == :wait_readable
+
+          if @clients.size >= MAX_CLIENTS
+            socket.close
+          else
+            @clients[socket] = { input: +"".b, output: nil, deadline: monotonic + CLIENT_TIMEOUT }
+          end
+        rescue Errno::ECONNABORTED
+          next
+        end
+        @clients.each do |socket, client|
+          if monotonic >= client[:deadline]
+            drop(socket)
+            next
+          end
+          read_request(socket, client) unless client[:output]
+          write_response(socket, client) if client[:output]
+        rescue IOError, SystemCallError
+          drop(socket)
+        end
+      end
+
+      def close
+        @clients.each_key(&:close)
+        @clients.clear
+        @io.close unless @io.closed?
+      end
+
+      private
+
+      def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+      def drop(socket)
+        @clients.delete(socket)
+        socket.close unless socket.closed?
+      end
+
+      def read_request(socket, client)
+        chunk = socket.read_nonblock(4096, exception: false)
+        return if chunk == :wait_readable
+        return drop(socket) if chunk.nil?
+
+        client[:input] << chunk
+        if client[:input].bytesize > MAX_HEADER_BYTES
+          respond(client, 431, "Request headers too large\n")
+        elsif client[:input].include?("\r\n\r\n")
+          method, path, version = client[:input].lines.first.split
+          if !%w[HTTP/1.0 HTTP/1.1].include?(version)
+            respond(client, 400, "Invalid request\n")
+          elsif method != "GET"
+            respond(client, 405, "GET required\n")
+          else
+            route(client, path)
+          end
+        end
+      end
+
+      def route(client, path)
+        case path
+        when "/livez" then respond(client, 200, "ok\n")
+        when "/readyz"
+          ready = @ready.call
+          respond(client, ready ? 200 : 503, ready ? "ready\n" : "not ready\n")
+        when "/status" then respond(client, 200, "#{JSON.generate(@status.call)}\n", "application/json")
+        when "/metrics" then respond(client, 200, @metrics.call, "text/plain; version=0.0.4")
+        else respond(client, 404, "Not found\n")
+        end
+      rescue StandardError => e
+        @logger.error("Admin #{path}: #{e.message}")
+        respond(client, 500, "Internal error\n")
+      end
+
+      def respond(client, code, body, content_type = "text/plain")
+        reasons = { 200 => "OK", 400 => "Bad Request", 404 => "Not Found", 405 => "Method Not Allowed",
+                    431 => "Request Header Fields Too Large", 500 => "Internal Server Error", 503 => "Service Unavailable" }
+        client[:output] =
+          "HTTP/1.1 #{code} #{reasons.fetch(code)}\r\nContent-Type: #{content_type}\r\nContent-Length: #{body.bytesize}\r\nConnection: close\r\n\r\n#{body}"
+      end
+
+      def write_response(socket, client)
+        written = socket.write_nonblock(client[:output], exception: false)
+        return if written == :wait_writable
+
+        client[:output] = client[:output].byteslice(written..)
+        drop(socket) if client[:output].empty?
+      end
+    end
+  end
+end