tonic-rails 0.5.0 → 0.7.0

data/ext/tonic_rails_native/src/gvl.rs

@@ -20,7 +20,7 @@ use magnus::{
     Error, RHash, RString, Ruby, Value, prelude::*, r_hash::ForEach, typed_data::DataTypeFunctions,
     value::Opaque,
 };
-use parking_lot::Mutex;
+use parking_lot::{Condvar, Mutex};
 use rb_sys::{RB_NOGVL_INTR_FAIL, rb_nogvl, rb_thread_call_with_gvl};
 use tokio::sync::{Notify, mpsc};
 
@@ -100,25 +100,40 @@ where
 
 /// Minimal single-future executor that parks the current (non-tokio)
 /// thread.  tokio's channel and `Notify` futures are runtime-agnostic.
+///
+/// It parks on a pthread condition variable rather than
+/// `std::thread::park`, which on macOS uses libdispatch: libdispatch
+/// traps in the child of a multi-threaded process, and the client blocks
+/// here in processes forked after it made calls.
 pub fn block_on<F: Future>(fut: F) -> F::Output {
-    struct ThreadWaker(std::thread::Thread);
-    impl Wake for ThreadWaker {
+    #[derive(Default)]
+    struct Signal {
+        woken: Mutex<bool>,
+        condvar: Condvar,
+    }
+    impl Wake for Signal {
         fn wake(self: Arc<Self>) {
-            self.0.unpark();
+            self.wake_by_ref();
         }
         fn wake_by_ref(self: &Arc<Self>) {
-            self.0.unpark();
+            *self.woken.lock() = true;
+            self.condvar.notify_one();
         }
     }
 
-    let waker = Waker::from(Arc::new(ThreadWaker(std::thread::current())));
+    let signal = Arc::new(Signal::default());
+    let waker = Waker::from(signal.clone());
     let mut cx = Context::from_waker(&waker);
     let mut fut = std::pin::pin!(fut);
     loop {
         if let Poll::Ready(value) = fut.as_mut().poll(&mut cx) {
             return value;
         }
-        std::thread::park();
+        let mut woken = signal.woken.lock();
+        while !*woken {
+            signal.condvar.wait(&mut woken);
+        }
+        *woken = false;
     }
 }
 

data/ext/tonic_rails_native/src/lib.rs

@@ -20,6 +20,7 @@
 //! Modules:
 //! - [`framing`]  — gRPC wire format (frames, gzip, timeouts, trailers)
 //! - [`body`]     — response bodies and request stats
+//! - [`client`]   — `TonicRails::Native::Channel` and `ResponseStream`, for outgoing calls
 //! - [`dispatch`] — the fallback tower service that queues RPCs for Ruby
 //! - [`gvl`]      — GVL release/acquire and the Ruby stream objects
 //! - [`log`]      — transport log events for Ruby's logger
@@ -27,6 +28,7 @@
 //! - [`server`]   — `TonicRails::Native::Server`
 
 mod body;
+mod client;
 mod dispatch;
 mod framing;
 mod gvl;
@@ -36,6 +38,7 @@ mod server;
 
 use magnus::{Error, Ruby, function, method, prelude::*};
 
+use client::{NativeChannel, ResponseStream};
 use gvl::{RequestReader, StreamSender};
 use server::NativeServer;
 
@@ -95,5 +98,16 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     reader.define_method("next", method!(RequestReader::next, 0))?;
     reader.define_method("error", method!(RequestReader::error, 0))?;
 
+    let channel = native.define_class("Channel", ruby.class_object())?;
+    channel.define_singleton_method("new", function!(NativeChannel::new, 4))?;
+    channel.define_method("unary", method!(NativeChannel::unary, 4))?;
+    channel.define_method("server_streaming", method!(NativeChannel::server_streaming, 4))?;
+
+    let stream = native.define_class("ResponseStream", ruby.class_object())?;
+    stream.undef_default_alloc_func();
+    stream.define_method("next", method!(ResponseStream::next, 0))?;
+    stream.define_method("result", method!(ResponseStream::result, 0))?;
+    stream.define_method("close", method!(ResponseStream::close, 0))?;
+
     Ok(())
 }

data/lib/tonic/rails/base.rb

@@ -248,7 +248,10 @@ module Tonic
 
         @request = build_request(reader, spec)
         response = nil
-        run_callbacks(:rpc) { response = respond(public_send(spec[:handler], @request), spec, sender) }
+        # Calls this handler makes with Tonic::Rails::Client default to its deadline.
+        Client.with_deadline(@grpc_deadline_at) do
+          run_callbacks(:rpc) { response = respond(public_send(spec[:handler], @request), spec, sender) }
+        end
         response
       rescue RpcError
         raise

data/lib/tonic/rails/client/retry_policy.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Tonic
+  module Rails
+    class Client
+      # When and how often a call is retried, following gRPC's retry design
+      # (A6).  Pass one, a Hash of its fields or `true` for the defaults as
+      # `retries:` to Client.new or to a single call.
+      #
+      #   Tonic::Rails::Client.new("items.internal:50051", retries: true)
+      #   Tonic::Rails::Client.new("items.internal:50051", retries: { max_attempts: 4 })
+      #   items.get_item({ id: 1 }, retries: false)
+      #
+      # Only calls the server hasn't started on are retried: the connection
+      # failed, or the server answered with a status and no response
+      # headers.  Each wait is random, up to a cap that grows from
+      # `initial_backoff` by `multiplier` per attempt to `max_backoff`.
+      # Retrying stops when the next attempt couldn't start before the
+      # call's deadline.
+      RetryPolicy = Data.define(:max_attempts, :codes, :initial_backoff, :max_backoff, :multiplier) do
+        # Defaults for the fields left out.
+        def self.defaults
+          { max_attempts: 3, codes: [:unavailable], initial_backoff: 0.1, max_backoff: 1, multiplier: 2 }
+        end
+
+        # @param option [RetryPolicy, Hash, true, false, nil]
+        # @return [RetryPolicy, nil] nil for no retries
+        def self.from(option)
+          case option
+          when nil, false then nil
+          when true then new(**defaults)
+          when Hash then new(**defaults, **option)
+          when self then option
+          else raise ArgumentError, "retries must be true, false, a Hash or a #{name}, got #{option.class}"
+          end
+        end
+
+        # @param max_attempts [Integer] attempts in all, from 2 to 5.  gRPC
+        #   caps it at 5, so a misconfigured client can't multiply an
+        #   outage's traffic by more than that.
+        # @param codes [Array<Symbol, Integer>] status codes worth retrying
+        # @param initial_backoff [Numeric] seconds; the cap on the first wait
+        # @param max_backoff [Numeric] seconds; the cap on any wait
+        # @param multiplier [Numeric] how much the cap grows per attempt
+        def initialize(max_attempts:, codes:, initial_backoff:, max_backoff:, multiplier:)
+          unless max_attempts.is_a?(Integer) && max_attempts.between?(2, 5)
+            raise ArgumentError, "max_attempts must be an Integer from 2 to 5"
+          end
+
+          codes = Array(codes).map { |code| RpcError.normalize_code(code) }.uniq.freeze
+          raise ArgumentError, "codes must name at least one status code other than OK" if codes.empty? || codes.include?(0)
+
+          [[:initial_backoff, initial_backoff], [:max_backoff, max_backoff]].each do |field, value|
+            raise ArgumentError, "#{field} must be a positive number of seconds" unless value.is_a?(Numeric) && value.positive?
+          end
+          raise ArgumentError, "multiplier must be at least 1" unless multiplier.is_a?(Numeric) && multiplier >= 1
+
+          super
+        end
+
+        # @param code [Integer]
+        def retry?(code)
+          codes.include?(code)
+        end
+
+        # Seconds to wait after failed attempt number `attempt` (from 1).
+        def backoff(attempt)
+          rand * [initial_backoff * (multiplier**(attempt - 1)), max_backoff].min
+        end
+      end
+    end
+  end
+end

data/lib/tonic/rails/client.rb

@@ -0,0 +1,460 @@
+# frozen_string_literal: true
+
+require "active_support/inflector"
+
+module Tonic
+  module Rails
+    # Calls other gRPC services, without the grpc gem.
+    #
+    #   ITEMS = Tonic::Rails::Client.new("items.internal:50051", deadline: 2)
+    #
+    #   items = ITEMS.stub("demo.v1.ItemService")
+    #   item = items.get_item(Demo::V1::GetItemRequest.new(id: 1))
+    #   item = items.get_item(id: 1)
+    #   item = items.get_item({ id: 1 }, deadline: 0.5, metadata: { "x-request-id" => request_id })
+    #   items.list_items(category: "books").each { |item| ... }
+    #
+    # Retries are opt-in: `retries: true` (or a Hash; see RetryPolicy) on
+    # the client or a call retries calls the server never started on.
+    #
+    # Stubs are built from the protobuf descriptors that
+    # `tonic_rails:compile_protos` (protoc --ruby_out) generates; no
+    # `_services_pb.rb` is needed.  A failed call raises
+    # Tonic::Rails::RpcError.
+    #
+    # A client holds one HTTP/2 connection and is safe to share between
+    # threads, so create it once (a constant or an initializer).  It keeps
+    # working across `fork`: each process connects on its own, with no
+    # hooks to call.
+    class Client
+      # A response with its headers and trailers; see `return_call:`.
+      Call = Data.define(:response, :headers, :trailers)
+
+      DEFAULT_MAX_MESSAGE_SIZE = 4 * 1024 * 1024
+
+      # Trailer a server sends to ask for a retry after so many
+      # milliseconds, or (with a negative value) for none.
+      RETRY_PUSHBACK = "grpc-retry-pushback-ms"
+      private_constant :RETRY_PUSHBACK
+
+      # @return [String] "host:port"
+      attr_reader :address
+
+      # @return [Numeric, nil] default deadline in seconds
+      attr_reader :deadline
+
+      # @return [RetryPolicy, nil] how calls are retried, unless they say otherwise
+      attr_reader :retries
+
+      # @param address [String] "host:port"
+      # @param tls [Boolean] false for plaintext, which must be explicit
+      # @param tls_ca_path [String, nil] CA certificates (PEM) to trust instead
+      #   of the system's root certificates
+      # @param tls_cert_path [String, nil] client certificate (PEM), for mutual TLS
+      # @param tls_key_path [String, nil] client private key (PEM), for mutual TLS
+      # @param tls_server_name [String, nil] name to verify the server's
+      #   certificate against, if not the host in `address`
+      # @param deadline [Numeric, nil] seconds each call may take, unless the
+      #   call sets its own
+      # @param metadata [Hash] sent with every call (see Tonic::Rails::Metadata)
+      # @param max_message_size [Integer] largest message sent or received, in bytes
+      # @param connect_timeout [Numeric] seconds to wait for a connection
+      # @param retries [RetryPolicy, Hash, Boolean] retry calls the server
+      #   never started on (see RetryPolicy); off by default
+      # @param logger [Logger, nil] where retries are logged; defaults to
+      #   Tonic::Rails.configuration.logger
+      def initialize(address, tls: true, tls_ca_path: nil, tls_cert_path: nil, tls_key_path: nil,
+                     tls_server_name: nil, deadline: nil, metadata: {}, max_message_size: DEFAULT_MAX_MESSAGE_SIZE,
+                     connect_timeout: 5, retries: false, logger: nil)
+        Tonic::Rails.native_extension!
+        validate_address!(address)
+        validate_tls!(tls, tls_cert_path, tls_key_path)
+        positive!(:deadline, deadline) unless deadline.nil?
+        positive!(:connect_timeout, connect_timeout)
+        unless max_message_size.is_a?(Integer) && max_message_size.positive?
+          raise ArgumentError, "max_message_size must be a positive Integer"
+        end
+
+        @address = address
+        @deadline = deadline
+        @retries = RetryPolicy.from(retries)
+        @logger = logger
+        @metadata = Metadata.encode(metadata)
+        @stubs = {}
+        tls_options = [read_pem(tls_ca_path, "CA"), read_pem(tls_cert_path, "certificate"),
+                       read_pem(tls_key_path, "key"), tls_server_name]
+        @native = TonicRails::Native::Channel.new(
+          "#{tls ? 'https' : 'http'}://#{address}", max_message_size, (connect_timeout * 1000).ceil,
+          (tls_options if tls)
+        )
+      end
+
+      # A stub with one method per RPC of the service, named in snake_case
+      # (`GetItem` becomes `get_item`).
+      #
+      # @param service [String, Google::Protobuf::ServiceDescriptor, #service_name]
+      #   the fully-qualified service name, its descriptor, or a grpc gem
+      #   service class (`Demo::V1::ItemService::Service`)
+      # @return [Tonic::Rails::Client::Stub]
+      def stub(service)
+        descriptor = service_descriptor(service)
+        @stubs[descriptor.name] ||= Stub.for(descriptor).new(self)
+      end
+
+      def inspect
+        "#<#{self.class.name} #{address}>"
+      end
+
+      # The deadline of the RPC this thread is handling, as a monotonic
+      # clock time; outgoing calls default to it.
+      # @api private
+      def self.current_deadline
+        Thread.current[:tonic_rails_deadline]
+      end
+
+      # @api private
+      def self.with_deadline(deadline)
+        previous = Thread.current[:tonic_rails_deadline]
+        Thread.current[:tonic_rails_deadline] = deadline
+        yield
+      ensure
+        Thread.current[:tonic_rails_deadline] = previous
+      end
+
+      # Make a unary call.  Stubs call this; use them instead.
+      # @api private
+      def unary(path, input, output, request, deadline: nil, metadata: {}, return_call: false, retries: nil)
+        bytes = input.encode(request_message(path, input, request))
+        metadata = call_metadata(metadata)
+        deadline_at = deadline_at(deadline)
+        policy = retry_policy(retries)
+
+        Instrumentation.instrument_request(path, address, metadata) do |payload|
+          code, error_message, response, headers, trailers, details = with_retries(path, policy, deadline_at, payload) do
+            @native.unary(path, bytes, metadata, remaining_ms(deadline_at))
+          end
+          headers = decode_metadata(headers)
+          trailers = decode_metadata(trailers)
+          check_status!(code, error_message, trailers, details)
+
+          response = decode_response(output, response, path)
+          return_call ? Call.new(response: response, headers: headers, trailers: trailers) : response
+        end
+      end
+
+      # Start a server-streaming call.  Stubs call this; use them instead.
+      # @api private
+      def server_streaming(path, input, output, request, deadline: nil, metadata: {}, retries: nil)
+        bytes = input.encode(request_message(path, input, request))
+        metadata = call_metadata(metadata)
+        # Fixed now, so the stream keeps the deadline of the RPC this
+        # thread is handling even if it's read later.
+        deadline_at = deadline_at(deadline)
+        policy = retry_policy(retries)
+
+        ServerStream.new(path) do |stream, block|
+          Instrumentation.instrument_request(path, address, metadata) do |payload|
+            # A message arrives only after the response headers, which
+            # commit the call, so an attempt that yielded anything is never
+            # retried.
+            code, error_message, _, headers, trailers, details = with_retries(path, policy, deadline_at, payload) do
+              read_stream(@native.server_streaming(path, bytes, metadata, remaining_ms(deadline_at)), output, path, &block)
+            end
+            trailers = decode_metadata(trailers)
+            stream.finish(decode_metadata(headers), trailers)
+            check_status!(code, error_message, trailers, details)
+          end
+        end
+      end
+
+      private
+
+      def validate_address!(address)
+        return if address.is_a?(String) && address.match?(%r{\A[^\s/]+:\d+\z})
+
+        raise ArgumentError, "address must be \"host:port\", got #{address.inspect}"
+      end
+
+      def validate_tls!(tls, cert_path, key_path)
+        raise ArgumentError, "tls must be true or false" unless [true, false].include?(tls)
+        raise ArgumentError, "tls_cert_path and tls_key_path must be set together" if cert_path.nil? != key_path.nil?
+        raise ArgumentError, "tls_cert_path and tls_key_path need tls: true" if cert_path && !tls
+      end
+
+      def positive!(name, value)
+        raise ArgumentError, "#{name} must be a positive number of seconds" unless value.is_a?(Numeric) && value.positive?
+      end
+
+      def read_pem(path, what)
+        return unless path
+        raise ArgumentError, "TLS #{what} file not found: #{path}" unless File.file?(path)
+
+        File.read(path)
+      end
+
+      def service_descriptor(service)
+        return service if service.is_a?(Google::Protobuf::ServiceDescriptor)
+
+        name = service.respond_to?(:service_name) ? service.service_name : service.to_s
+        descriptor = Google::Protobuf::DescriptorPool.generated_pool.lookup(name)
+        return descriptor if descriptor.is_a?(Google::Protobuf::ServiceDescriptor)
+
+        raise NotConfiguredError,
+              "#{name} is not a known protobuf service. Require its generated _pb.rb file (protoc --ruby_out) first."
+      end
+
+      def request_message(path, input, request)
+        case request
+        when input then request
+        when Hash then input.new(request)
+        else raise TypeError, "#{path} expects a #{input} or Hash, got #{request.class}"
+        end
+      end
+
+      def call_metadata(metadata)
+        metadata.empty? ? @metadata : @metadata.merge(Metadata.encode(metadata))
+      end
+
+      # When the call must end, as a monotonic clock time: the call's own
+      # deadline, or else the earlier of the client's default and the
+      # deadline of the RPC being handled.
+      def deadline_at(deadline)
+        now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        case deadline
+        when nil then [(now + @deadline if @deadline), self.class.current_deadline].compact.min
+        when Numeric then now + deadline
+        when Time then now + (deadline - Time.now)
+        else raise ArgumentError, "deadline must be seconds (Numeric) or a Time, got #{deadline.class}"
+        end
+      end
+
+      # Milliseconds left until `deadline_at`, or nil for no deadline.
+      def remaining_ms(deadline_at)
+        return unless deadline_at
+
+        remaining = deadline_at - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        raise RpcError.new("deadline exceeded", code: :deadline_exceeded) unless remaining.positive?
+
+        (remaining * 1000).ceil
+      end
+
+      def retry_policy(retries)
+        retries.nil? ? @retries : RetryPolicy.from(retries)
+      end
+
+      # Yield each message of a native stream, then return its result.
+      def read_stream(native, output, path)
+        while (bytes = native.next)
+          yield decode_response(output, bytes, path)
+        end
+        native.result
+      ensure
+        native.close
+      end
+
+      # Run attempts of a call (the block returns a native result) until one
+      # succeeds or shouldn't be retried; return its result.
+      def with_retries(path, policy, deadline_at, payload)
+        attempt = 1
+        loop do
+          payload[:attempts] = attempt if payload
+          result = yield
+          delay = retry_delay(policy, attempt, result, deadline_at)
+          if delay.nil?
+            log_gave_up(path, attempt, result) if attempt > 1 && !result[0].zero?
+            return result
+          end
+
+          log_retry(path, attempt, policy.max_attempts, result, delay)
+          sleep(delay)
+          attempt += 1
+        end
+      end
+
+      # Seconds to wait before retrying, or nil not to retry.  The server
+      # can ask for a delay, or for no retry, with grpc-retry-pushback-ms.
+      def retry_delay(policy, attempt, result, deadline_at)
+        code, _, _, _, trailers, _, committed = result
+        return if policy.nil? || code.zero? || committed || attempt >= policy.max_attempts || !policy.retry?(code)
+
+        pushback = trailers.assoc(RETRY_PUSHBACK)&.last
+        delay = if pushback.nil? then policy.backoff(attempt)
+        elsif pushback.match?(/\A\d+\z/) then pushback.to_i / 1000.0
+        end
+        return if delay.nil?
+        return if deadline_at && Process.clock_gettime(Process::CLOCK_MONOTONIC) + delay >= deadline_at
+
+        delay
+      end
+
+      def logger
+        @logger || Tonic::Rails.configuration.logger
+      end
+
+      def log_retry(path, attempt, max_attempts, result, delay)
+        logger&.warn(format("tonic-rails: %<path>s failed with %<status>s (attempt %<attempt>d of %<max>d); " \
+                            "retrying in %<ms>dms", path: path, status: status_text(result), attempt: attempt,
+                                                    max: max_attempts, ms: (delay * 1000).round))
+      end
+
+      def log_gave_up(path, attempts, result)
+        logger&.warn("tonic-rails: #{path} failed with #{status_text(result)} after #{attempts} attempts")
+      end
+
+      def status_text(result)
+        code, message = result
+        text = RpcError::CODE_NAMES.fetch(code, code).to_s.upcase
+        message.empty? ? text : "#{text}: #{message}"
+      end
+
+      def check_status!(code, message, trailers, details)
+        return if code.zero?
+
+        raise RpcError.new(message.empty? ? nil : message, code: code, metadata: trailers,
+                                                           details: RpcError.decode_details(details))
+      end
+
+      # Wire-form [name, value] pairs to a Hash: `-bin` values decoded,
+      # repeated names collected into an Array, and transport headers
+      # (content-type, grpc-*) dropped.
+      def decode_metadata(pairs)
+        pairs.each_with_object({}) do |(name, value), metadata|
+          next if name.start_with?("grpc-") || Metadata::RESERVED.include?(name)
+
+          value = value.unpack1("m") if name.end_with?("-bin")
+          metadata[name] = metadata.key?(name) ? [*metadata[name], value] : value
+        end
+      end
+
+      def decode_response(output, bytes, path)
+        output.decode(bytes)
+      rescue Google::Protobuf::ParseError
+        raise RpcError.new("invalid protobuf response from #{path}", code: :internal)
+      end
+
+      # Calls an RPC's methods.  Built per service from its descriptor; get
+      # one with Client#stub.
+      class Stub
+        class << self
+          # @api private
+          attr_reader :service_name
+
+          # @api private
+          def for(descriptor)
+            Class.new(self) do
+              @service_name = descriptor.name
+              descriptor.each { |method| define_rpc(method) }
+            end
+          end
+
+          def inspect
+            service_name ? "#{Stub.name}(#{service_name})" : super
+          end
+
+          private
+
+          def define_rpc(method)
+            path = "/#{service_name}/#{method.name}"
+            name = ActiveSupport::Inflector.underscore(method.name)
+            if method.client_streaming
+              define_method(name) do |*, **|
+                raise NotImplementedError,
+                      "#{path} is a client-streaming RPC; the tonic-rails client makes unary and server-streaming calls"
+              end
+              return
+            end
+
+            input = method.input_type.msgclass
+            output = method.output_type.msgclass
+            if method.server_streaming
+              define_method(name) do |request = nil, deadline: nil, metadata: {}, retries: nil, **fields|
+                request = rpc_request(name, request, fields)
+                @client.server_streaming(path, input, output, request, deadline: deadline, metadata: metadata,
+                                                                       retries: retries)
+              end
+            else
+              # `get_item(id: 1)` works as well as `get_item({ id: 1 })`.
+              define_method(name) do |request = nil, deadline: nil, metadata: {}, return_call: false, retries: nil,
+                                      **fields|
+                request = rpc_request(name, request, fields)
+                @client.unary(path, input, output, request, deadline: deadline, metadata: metadata,
+                                                            return_call: return_call, retries: retries)
+              end
+            end
+          end
+        end
+
+        # @return [Tonic::Rails::Client]
+        attr_reader :client
+
+        def initialize(client)
+          @client = client
+        end
+
+        def inspect
+          "#<#{self.class.inspect} #{client.address}>"
+        end
+
+        private
+
+        def rpc_request(name, request, fields)
+          unless request.nil? || fields.empty?
+            raise ArgumentError, "#{name}: pass the request or its fields, not both (#{fields.keys.join(', ')})"
+          end
+
+          request || fields
+        end
+      end
+
+      # The responses of a server-streaming call.  Nothing is sent until
+      # it's read; then messages are yielded as they arrive, a few at most
+      # buffered ahead of the reader.
+      #
+      #   items.list_items(category: "books").each { |item| ... }
+      #   items.list_items(category: "books").first(10)
+      #
+      # Stopping early (`break`, `first(n)`, an exception) cancels the call.
+      # An error from the server raises Tonic::Rails::RpcError after the
+      # messages that came before it.  A stream can be read once.
+      class ServerStream
+        include Enumerable
+
+        # @return [Hash, nil] response headers, once the stream has ended
+        attr_reader :headers
+
+        # @return [Hash, nil] trailing metadata, once the stream has ended
+        attr_reader :trailers
+
+        # @api private
+        def initialize(path, &read)
+          @path = path
+          @read = read
+        end
+
+        # @yieldparam response [Google::Protobuf::AbstractMessage]
+        # @return [self, Enumerator] an Enumerator without a block
+        def each(&block)
+          return enum_for(:each) unless block
+          raise Error, "the #{@path} stream has already been read" unless @read
+
+          read = @read
+          @read = nil
+          read.call(self, block)
+          self
+        end
+
+        # @api private
+        def finish(headers, trailers)
+          @headers = headers
+          @trailers = trailers
+        end
+
+        def inspect
+          "#<#{self.class.name} #{@path}>"
+        end
+      end
+    end
+  end
+end
+
+require "tonic/rails/client/retry_policy"

data/lib/tonic/rails/instrumentation.rb

@@ -24,18 +24,34 @@ module Tonic
     module Instrumentation
       EVENT = "process_rpc.tonic_rails"
 
+      # Outgoing calls made with Tonic::Rails::Client publish
+      # "request_rpc.tonic_rails", with the same payload plus :address, the
+      # server called, and :attempts, how many times it was tried.
+      REQUEST_EVENT = "request_rpc.tonic_rails"
+
       module_function
 
       # @api private
-      def instrument(path, metadata)
+      def instrument(path, metadata, &)
+        publish(EVENT, path, metadata, &)
+      end
+
+      # @api private
+      def instrument_request(path, address, metadata, &)
+        publish(REQUEST_EVENT, path, metadata, address: address, &)
+      end
+
+      # Yields the payload, or nil when nothing is listening.
+      # @api private
+      def publish(event, path, metadata, **extra)
         # Without subscribers, skip building the payload: this runs on
         # every RPC.
-        return yield unless ActiveSupport::Notifications.notifier.listening?(EVENT)
+        return yield(nil) unless ActiveSupport::Notifications.notifier.listening?(event)
 
         service, rpc = path.delete_prefix("/").split("/", 2)
-        payload = { path: path, service: service, rpc: rpc, metadata: metadata, code: nil, status: nil }
-        ActiveSupport::Notifications.instrument(EVENT, payload) do
-          result = yield
+        payload = { path: path, service: service, rpc: rpc, metadata: metadata, code: nil, status: nil, **extra }
+        ActiveSupport::Notifications.instrument(event, payload) do
+          result = yield(payload)
           payload[:code] = 0
           payload[:status] = :ok
           result