tonic-rails 0.6.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b39b4d814c854c16c5a1f9c7ec05415402fdbb49716846bc3883442931f5165
-  data.tar.gz: f4dcedf8d10ec7cd0786232a7637e192ec5dd7746ee15542322465d56bc11436
+  metadata.gz: d6cc5ee93380e004201e88f63289b88dff01dd6cc6566c92a3d972b9c934150d
+  data.tar.gz: 20a5a45adb019fa7275330b2d5b9f256a9f259765e1ffc83f4f1278902b38620
 SHA512:
-  metadata.gz: b1b5406e16afb900514ae85ae11350d7a7546c0a08ed27c3c1042622de85b4ea9ac692185e3a0d27792f95ca36e61a1f2ff0b5729bf8e38eb645ed9e45c011ee
-  data.tar.gz: 7b3b7386f1c8342cf0c25fce33ea1564c38e1be027788b7b919377354b9a9a53f3c58338e334185d8dd0377f261b9775fadeee1cc4a2b0ac627764737329fd2b
+  metadata.gz: 452568660e7a6f19c16d77ff8d272691c0f54c7ebf67261e5199dab6ad59f6c1df89b4b8e0efab84748e089bd74f4e4c2bf4de7dd599e628de0fc11f6ce03922
+  data.tar.gz: 52d5199c4d503c0792d84d254442a87cc25fc962b9af87561129c08c95974c48424c1a8914bd0cfcf664331d5c2f892c6cd8f31e507309bacb8c32b3ecc2fb53

data/CHANGELOG.md

@@ -7,6 +7,47 @@ minor versions may contain breaking changes.
 
 ## [Unreleased]
 
+## [0.8.0] - 2026-10-12
+
+### Added
+
+- Client: `method_retries:` sets retry policies per service
+  (`"demo.v1.ItemService"`) or per method
+  (`"demo.v1.ItemService/GetItem"`), like gRPC's service config. The most
+  specific setting wins: the call's `retries:`, then the method, then the
+  service, then the client's `retries:`.
+
+### Fixed
+
+- Windows: the process could crash with a segfault under load (#41). The
+  extension's locks spun and then called `Sleep`, which the mingw linker
+  bound to the `Sleep` that Ruby's DLL exports instead of the system's,
+  and Ruby's version crashes on threads Ruby doesn't own. Locks now come
+  from Rust's standard library, which doesn't spin through `Sleep`, the
+  build binds `Sleep` to kernel32, and CI checks that the Windows build
+  imports nothing but Ruby's C API from Ruby's DLL. 0.6.0 and 0.7.0 crashed in about 1 in 5 runs of the test suite;
+  earlier versions could crash in the same way, but rarely.
+
+## [0.7.0] - 2026-10-11
+
+### Added
+
+- Client: opt-in retries, per client or per call (`retries: true` or a
+  Hash of `max_attempts`, `codes`, `initial_backoff`, `max_backoff` and
+  `multiplier`), following gRPC's retry design. Only calls the server
+  never started on are retried: a failed connection, or a status sent
+  without response headers. Waits are random with exponential caps, stop
+  before the deadline and follow `grpc-retry-pushback-ms`. Each retry is
+  logged, and `request_rpc.tonic_rails` reports `:attempts`. Closes #39.
+
+### Changed
+
+- The server answers calls it turns away before a handler runs (shutting
+  down, dispatch queue full, malformed requests) with a Trailers-Only
+  response: the status in the headers and no body. Clients that follow
+  gRPC's retry rules can tell those calls never started and retry them
+  safely.
+
 ## [0.6.0] - 2026-10-11
 
 ### Added
@@ -166,7 +207,9 @@ First public release.
 - Precompiled native gems for Linux (x86_64, aarch64; glibc and musl) and
   macOS (x86_64, arm64), Ruby 3.3 to 4.0.
 
-[Unreleased]: https://github.com/CodingAnarchy/tonic-rails/compare/v0.6.0...HEAD
+[Unreleased]: https://github.com/CodingAnarchy/tonic-rails/compare/v0.8.0...HEAD
+[0.8.0]: https://github.com/CodingAnarchy/tonic-rails/compare/v0.7.0...v0.8.0
+[0.7.0]: https://github.com/CodingAnarchy/tonic-rails/compare/v0.6.0...v0.7.0
 [0.6.0]: https://github.com/CodingAnarchy/tonic-rails/compare/v0.5.0...v0.6.0
 [0.5.0]: https://github.com/CodingAnarchy/tonic-rails/compare/v0.4.0...v0.5.0
 [0.4.0]: https://github.com/CodingAnarchy/tonic-rails/compare/v0.3.0...v0.4.0

data/Cargo.lock

@@ -468,15 +468,6 @@ dependencies = [
  "windows-link",
 ]
 
-[[package]]
-name = "lock_api"
-version = "0.4.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
-dependencies = [
- "scopeguard",
-]
-
 [[package]]
 name = "log"
 version = "0.4.34"
@@ -573,29 +564,6 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
 
-[[package]]
-name = "parking_lot"
-version = "0.12.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
-dependencies = [
- "lock_api",
- "parking_lot_core",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.9.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
-dependencies = [
- "cfg-if",
- "libc",
- "redox_syscall",
- "smallvec",
- "windows-link",
-]
-
 [[package]]
 name = "percent-encoding"
 version = "2.3.2"
@@ -708,15 +676,6 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cca7ad6a7e21e72151d56fe2495a259b5670e204c3adac41ee7ef676ea08117a"
 
-[[package]]
-name = "redox_syscall"
-version = "0.5.18"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
-dependencies = [
- "bitflags",
-]
-
 [[package]]
 name = "regex"
 version = "1.13.1"
@@ -822,12 +781,6 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
-[[package]]
-name = "scopeguard"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
-
 [[package]]
 name = "security-framework"
 version = "3.7.0"
@@ -1107,7 +1060,7 @@ dependencies = [
 
 [[package]]
 name = "tonic_rails_native"
-version = "0.6.0"
+version = "0.8.0"
 dependencies = [
  "axum",
  "base64",
@@ -1118,7 +1071,6 @@ dependencies = [
  "http-body",
  "http-body-util",
  "magnus",
- "parking_lot",
  "prost",
  "prost-types",
  "rb-sys",

data/README.md

@@ -507,6 +507,86 @@ If a handler lets the error propagate, its caller receives the same
 status, message, details and trailing metadata. Rescue it and raise your
 own error when those are internal.
 
+### Retries
+
+Retries are off unless you turn them on, for a client or a single call:
+
+```ruby
+ITEMS = Tonic::Rails::Client.new("items.internal:50051", deadline: 2, retries: true)
+
+items.get_item({ id: 1 }, retries: false)                    # not this call
+items.get_item({ id: 1 }, retries: { max_attempts: 5 })      # or differently
+```
+
+Different methods often need different policies: a read can retry more
+eagerly than a write. `method_retries:` sets them per service or per
+method, by their protobuf names:
+
+```ruby
+ITEMS = Tonic::Rails::Client.new(
+  "items.internal:50051",
+  retries: true,                                                   # every call...
+  method_retries: {
+    "demo.v1.ItemService/GetItem" => { max_attempts: 5 },          # ...but this method,
+    "demo.v1.AdminService" => false,                               # and this whole service
+  },
+)
+```
+
+The most specific setting wins: the call's own `retries:`, then its
+method's entry, then its service's, then the client's `retries:`.
+
+`retries: true` tries a call up to 3 times when it fails with
+`UNAVAILABLE`. A Hash changes any of these:
+
+| Field | Default | |
+|---|---|---|
+| `max_attempts` | 3 | attempts in all, from 2 to 5 |
+| `codes` | `[:unavailable]` | status codes worth retrying |
+| `initial_backoff` | 0.1 | seconds; the cap on the first wait |
+| `max_backoff` | 1 | seconds; the cap on any wait |
+| `multiplier` | 2 | how much the cap grows per attempt |
+
+The rules come from gRPC's own retry design, so they avoid the usual
+mistakes of a hand-written retry loop:
+
+- **A call is retried only if the server never started on it.** That
+  means the connection failed (refused, reset, a pod restarting), or the
+  server answered with only a status and no response headers. Servers do
+  that when they turn a call away before any handler runs: a
+  tonic-rails server that's shutting down or whose queue is full, or a
+  proxy with no healthy backend. Once response headers arrive, the call
+  is never repeated, even if it then fails with a code you listed.
+- **Waits are random.** Each one is anywhere from zero up to a cap that
+  starts at `initial_backoff` and grows by `multiplier` per attempt, up to
+  `max_backoff`. Clients that failed together don't retry together.
+- **The deadline covers every attempt.** If the next attempt couldn't
+  start before the deadline, the client stops and raises the last
+  failure.
+- **The server can steer.** A `grpc-retry-pushback-ms` trailer sets the
+  wait, and a negative one stops retries.
+- A server stream is retried only before its first message.
+
+Which calls are safe to retry? With the default `[:unavailable]`, nearly
+all of them. By gRPC convention, `UNAVAILABLE` means the call can be tried
+again, and from a tonic-rails server it's retried only when no handler
+ran. For a tonic-rails server, `:resource_exhausted` is worth adding too:
+a full dispatch queue returns it before any handler runs.
+
+Take more care with codes a handler might raise after doing some work,
+like `:aborted`. A tonic-rails server sends headers before it runs a
+handler, so those calls still won't be retried. Other servers may answer
+with only a status after a handler ran, so list those codes only for
+methods that are safe to run twice.
+
+Each retry is logged as a warning, through the client's `logger:` (by
+default `Tonic::Rails.configuration.logger`, which is the Rails logger in
+a Rails app):
+
+```
+tonic-rails: /demo.v1.ItemService/GetItem failed with UNAVAILABLE: ... (attempt 1 of 3); retrying in 64ms
+```
+
 ### Metadata
 
 `metadata:` on a call, and on the client for every call, works like
@@ -549,6 +629,9 @@ the client is created, not on the first call.
 | `metadata` | `{}` | sent with every call |
 | `max_message_size` | 4 MiB | largest message sent or received |
 | `connect_timeout` | 5 | seconds to wait for a connection |
+| `retries` | `false` | retry calls the server never started on; see [Retries](#retries) |
+| `method_retries` | `{}` | retries per service or method, by name |
+| `logger` | `Tonic::Rails.configuration.logger` | where retries are logged |
 
 ### Fork safety
 
@@ -830,7 +913,8 @@ When a call fails, `event.payload[:exception_object]` is the
 unexpected, that error's `cause` is the original exception.
 
 Calls made with `Tonic::Rails::Client` publish `request_rpc.tonic_rails`,
-with the same payload plus `:address`, the server called. For a
+with the same payload plus `:address`, the server called, and
+`:attempts`, how many times it was tried. For a
 server-streaming call, the event covers the whole stream, including the
 time your block spends on each message.
 

data/ext/tonic_rails_native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tonic_rails_native"
-version = "0.6.0"
+version = "0.8.0"
 edition = "2024"
 rust-version = "1.88"
 publish = false
@@ -41,7 +41,6 @@ tokio-stream = { version = "0.1", features = ["net"] }
 prost = "0.14"
 prost-types = "0.14"
 socket2 = { version = "0.6", features = ["all"] }
-parking_lot = "0.12"
 crossbeam-channel = "0.5"
 http = "1"
 http-body = "1"

data/ext/tonic_rails_native/extconf.rb

@@ -1,7 +1,35 @@
 # frozen_string_literal: true
 
+require "fileutils"
 require "mkmf"
 require "rb_sys/mkmf"
+require "shellwords"
+
+# On mingw, Ruby's DLL exports its own `Sleep`, which releases the GVL
+# around the wait: on a thread Ruby doesn't own it crashes, and on a Ruby
+# thread that has already released the GVL it takes the GVL back from
+# whichever thread holds it (#41).  rb-sys links Ruby's import library ahead of
+# kernel32, so the extension's `Sleep` (from mingw's DLL startup code, and
+# from std via crossbeam) would bind to Ruby's.  Link a one-function import
+# library binding `Sleep` to kernel32 ahead of Ruby's instead.  (Putting
+# all of kernel32 first goes too far: some toolchains' kernel32 import
+# libraries claim functions kernel32.dll doesn't export.)
+# script/check_windows_imports.rb checks the result in CI.
+if RbConfig::CONFIG["target_os"].to_s.include?("mingw")
+  dir = File.expand_path("kernel32_sleep")
+  FileUtils.mkdir_p(dir)
+  File.write(File.join(dir, "sleep.def"), "LIBRARY kernel32.dll\nEXPORTS\nSleep\n")
+  # Cross-compiling, RbConfig describes the target, so its DLLTOOL may not
+  # exist here: try the one next to the C compiler too.
+  compiler = Shellwords.split(RbConfig::CONFIG["CC"].to_s).find { |word| word.include?("gcc") }.to_s
+  candidates = [RbConfig::CONFIG["DLLTOOL"], compiler.sub(/gcc(\.exe)?\z/, "dlltool"), "dlltool"]
+  dlltool = candidates.reject { |tool| tool.to_s.empty? }.uniq.find { |tool| find_executable(tool) }
+  abort "tonic-rails: no dlltool found (tried #{candidates.compact.uniq.join(', ')})" unless dlltool
+  unless system(dlltool, "-d", File.join(dir, "sleep.def"), "-l", File.join(dir, "libtonic_rails_sleep.a"))
+    abort "tonic-rails: #{dlltool} failed to build the kernel32 Sleep import library"
+  end
+  RbConfig::CONFIG["LIBS"] = "-L#{Shellwords.escape(dir)} -ltonic_rails_sleep #{RbConfig::CONFIG['LIBS']}"
+end
 
 # Builds the Rust crate with cargo and installs the library as
 # lib/tonic/rails/tonic_rails_native.{so,bundle}, matching

data/ext/tonic_rails_native/src/body.rs

@@ -12,7 +12,6 @@ use std::task::{Context, Poll};
 use std::time::Instant;
 
 use bytes::Bytes;
-use http::HeaderMap;
 use http_body::{Body, Frame};
 use tokio::sync::mpsc;
 use tonic::{Code, Status};
@@ -71,35 +70,6 @@ impl ServerStats {
     }
 }
 
-/// Trailers-only error response for requests rejected before dispatch.
-pub struct ErrorBody {
-    trailers: Option<HeaderMap>,
-}
-
-impl ErrorBody {
-    pub fn new(code: Code, message: &str) -> Self {
-        Self {
-            trailers: Some(status_trailers(code as u32, message)),
-        }
-    }
-}
-
-impl Body for ErrorBody {
-    type Data = Bytes;
-    type Error = Status;
-
-    fn poll_frame(
-        mut self: Pin<&mut Self>,
-        _cx: &mut Context<'_>,
-    ) -> Poll<Option<Result<Frame<Bytes>, Status>>> {
-        Poll::Ready(self.trailers.take().map(|t| Ok(Frame::trailers(t))))
-    }
-
-    fn is_end_stream(&self) -> bool {
-        self.trailers.is_none()
-    }
-}
-
 /// Streaming response body fed by the Ruby handler through an mpsc
 /// channel.  Each `Data` message becomes a gRPC frame; `End` (or the
 /// deadline) produces the trailers.
@@ -232,6 +202,7 @@ impl Drop for ResponseBody {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use http::HeaderMap;
     use std::task::Waker;
 
     type PolledFrame = Poll<Option<Result<Frame<Bytes>, Status>>>;
@@ -262,16 +233,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn error_body_yields_trailers_once() {
-        let mut body = ErrorBody::new(Code::NotFound, "missing");
-        let t = trailers(poll(&mut body));
-        assert_eq!(t["grpc-status"], "5");
-        assert_eq!(t["grpc-message"], "missing");
-        assert!(matches!(poll(&mut body), Poll::Ready(None)));
-        assert!(body.is_end_stream());
-    }
-
     #[test]
     fn response_body_streams_data_then_ok() {
         let (tx, rx) = mpsc::channel(4);

data/ext/tonic_rails_native/src/client.rs

@@ -24,7 +24,6 @@ use bytes::{Buf, BufMut, Bytes};
 use http::uri::PathAndQuery;
 use http::{HeaderMap, HeaderName, HeaderValue};
 use magnus::{Error, RArray, RHash, RString, Ruby, r_hash::ForEach};
-use parking_lot::Mutex;
 use tokio::runtime::{Builder, Handle, Runtime};
 use tokio::sync::mpsc;
 use tokio::task::JoinHandle;
@@ -34,6 +33,7 @@ use tonic::transport::{Certificate, Channel, ClientTlsConfig, Endpoint, Identity
 use tonic::{Code, Request, Status};
 
 use crate::gvl::{Cancel, Wait, block_on, recv_blocking, without_gvl};
+use crate::sync::Mutex;
 
 // ─── Per-process runtime ──────────────────────────────────────────
 
@@ -124,6 +124,9 @@ struct Outcome {
     /// The failure happened here (a broken connection, a timeout), not
     /// on the server.
     local: bool,
+    /// Response headers arrived, so the server has started on the call
+    /// and retrying it could repeat its work.
+    committed: bool,
     details: Bytes,
     response: Option<Bytes>,
     headers: HeaderMap,
@@ -136,6 +139,7 @@ impl Outcome {
             code: Code::Ok,
             message: String::new(),
             local: false,
+            committed: true,
             details: Bytes::new(),
             response,
             headers,
@@ -146,11 +150,13 @@ impl Outcome {
     fn error(code: Code, message: impl Into<String>) -> Self {
         Self {
             local: true,
-            ..Self::status(Status::new(code, message), HeaderMap::new())
+            ..Self::status(Status::new(code, message), None)
         }
     }
 
-    fn status(status: Status, headers: HeaderMap) -> Self {
+    /// `headers` are the response headers, if they arrived before the
+    /// status.
+    fn status(status: Status, headers: Option<HeaderMap>) -> Self {
         // A status with a source is a local transport failure, not one the
         // server sent.  The source holds the cause (e.g. "Connection
         // refused"), and tonic reports broken connections as UNKNOWN or
@@ -170,24 +176,26 @@ impl Outcome {
             code,
             message,
             local,
+            committed: headers.is_some(),
             details: Bytes::copy_from_slice(status.details()),
             response: None,
-            headers,
+            headers: headers.unwrap_or_default(),
             trailers: status.metadata().clone().into_headers(),
         }
     }
 
-    /// `[code, message, response, headers, trailers, details]`, with
-    /// metadata as `[name, value]` pairs in wire form (`-bin` values
+    /// `[code, message, response, headers, trailers, details, committed]`,
+    /// with metadata as `[name, value]` pairs in wire form (`-bin` values
     /// base64-encoded).
     fn into_ruby(self, ruby: &Ruby) -> Result<RArray, Error> {
-        let result = ruby.ary_new_capa(6);
+        let result = ruby.ary_new_capa(7);
         result.push(self.code as u32)?;
         result.push(ruby.str_new(&self.message))?;
         result.push(self.response.map(|bytes| ruby.str_from_slice(&bytes)))?;
         result.push(metadata_pairs(ruby, &self.headers)?)?;
         result.push(metadata_pairs(ruby, &self.trailers)?)?;
         result.push((!self.details.is_empty()).then(|| ruby.str_from_slice(&self.details)))?;
+        result.push(self.committed)?;
         Ok(result)
     }
 }
@@ -252,8 +260,10 @@ async fn start(args: CallArgs) -> Result<(HeaderMap, Streaming<Bytes>), Box<Outc
             let (headers, stream, _) = response.into_parts();
             Ok((headers.into_headers(), stream))
         }
-        // Includes trailers-only responses, whose status comes in the headers.
-        Err(status) => Err(Box::new(Outcome::status(status, HeaderMap::new()))),
+        // Includes trailers-only responses, whose status comes in the
+        // headers.  They don't commit the call: gRPC retries treat a server
+        // that answers with only a status as not having started on it.
+        Err(status) => Err(Box::new(Outcome::status(status, None))),
     }
 }
 
@@ -261,7 +271,7 @@ async fn start(args: CallArgs) -> Result<(HeaderMap, Streaming<Bytes>), Box<Outc
 async fn finish(stream: &mut Streaming<Bytes>, headers: &HeaderMap) -> Result<HeaderMap, Box<Outcome>> {
     match stream.trailers().await {
         Ok(trailers) => Ok(trailers.map(MetadataMap::into_headers).unwrap_or_default()),
-        Err(status) => Err(Box::new(Outcome::status(status, headers.clone()))),
+        Err(status) => Err(Box::new(Outcome::status(status, Some(headers.clone())))),
     }
 }
 
@@ -272,7 +282,7 @@ async fn unary(args: CallArgs) -> Outcome {
     };
     let message = match stream.message().await {
         Ok(message) => message,
-        Err(status) => return Outcome::status(status, headers),
+        Err(status) => return Outcome::status(status, Some(headers)),
     };
     let trailers = match finish(&mut stream, &headers).await {
         Ok(trailers) => trailers,
@@ -300,7 +310,7 @@ async fn server_stream(args: CallArgs, tx: mpsc::Sender<Bytes>) -> Outcome {
                 }
             }
             Ok(None) => break,
-            Err(status) => return Outcome::status(status, headers),
+            Err(status) => return Outcome::status(status, Some(headers)),
         }
     }
     match finish(&mut stream, &headers).await {

data/ext/tonic_rails_native/src/dispatch.rs

@@ -23,8 +23,8 @@ use tonic::Code;
 use tonic::body::Body as TonicBody;
 use tonic::transport::server::{TcpConnectInfo, TlsConnectInfo};
 
-use crate::body::{ErrorBody, ResponseBody, ServerStats};
-use crate::framing::{MessageReader, append_metadata, parse_grpc_timeout};
+use crate::body::{ResponseBody, ServerStats};
+use crate::framing::{MessageReader, append_metadata, parse_grpc_timeout, status_trailers};
 use crate::gvl::{DispatchContext, Peer, RequestError, StreamMessage};
 use crate::log::Level;
 
@@ -84,9 +84,16 @@ fn grpc_response(body: TonicBody, compressed: bool) -> Response<TonicBody> {
 }
 
 impl DispatchInner {
+    /// A Trailers-Only response (the status in the headers, no body) for a
+    /// call rejected before it reached a handler.  Clients may retry those
+    /// safely, and gRPC retries count on the form to tell them apart.
     fn reject(&self, path: &str, start: Instant, code: Code, message: &str) -> Response<TonicBody> {
         self.stats.record(path, code as u32, message, start, true);
-        grpc_response(TonicBody::new(ErrorBody::new(code, message)), false)
+        let mut response = grpc_response(TonicBody::empty(), false);
+        response
+            .headers_mut()
+            .extend(status_trailers(code as u32, message));
+        response
     }
 
     async fn handle(&self, req: Request<axum::body::Body>) -> Response<TonicBody> {

data/ext/tonic_rails_native/src/gvl.rs

@@ -16,11 +16,11 @@ use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
 use std::task::{Context, Poll, Wake, Waker};
 
 use crate::log::{Level, LogSink};
+use crate::sync::{Condvar, Mutex};
 use magnus::{
     Error, RHash, RString, Ruby, Value, prelude::*, r_hash::ForEach, typed_data::DataTypeFunctions,
     value::Opaque,
 };
-use parking_lot::{Condvar, Mutex};
 use rb_sys::{RB_NOGVL_INTR_FAIL, rb_nogvl, rb_thread_call_with_gvl};
 use tokio::sync::{Notify, mpsc};
 
@@ -101,7 +101,7 @@ where
 /// Minimal single-future executor that parks the current (non-tokio)
 /// thread.  tokio's channel and `Notify` futures are runtime-agnostic.
 ///
-/// It parks on a pthread condition variable rather than
+/// It parks on a condition variable rather than
 /// `std::thread::park`, which on macOS uses libdispatch: libdispatch
 /// traps in the child of a multi-threaded process, and the client blocks
 /// here in processes forked after it made calls.
@@ -131,7 +131,7 @@ pub fn block_on<F: Future>(fut: F) -> F::Output {
         }
         let mut woken = signal.woken.lock();
         while !*woken {
-            signal.condvar.wait(&mut woken);
+            woken = signal.condvar.wait(woken);
         }
         *woken = false;
     }

data/ext/tonic_rails_native/src/lib.rs

@@ -35,6 +35,7 @@ mod gvl;
 mod log;
 mod net;
 mod server;
+mod sync;
 
 use magnus::{Error, Ruby, function, method, prelude::*};
 

data/ext/tonic_rails_native/src/server.rs

@@ -22,7 +22,6 @@ use std::time::{Duration, Instant};
 use magnus::{
     Error, RArray, RHash, RString, Ruby, TryConvert, Value, gc, typed_data::DataTypeFunctions, value::Opaque,
 };
-use parking_lot::Mutex;
 use prost::Message;
 use tokio::runtime::Runtime;
 use tokio::sync::{Semaphore, watch};
@@ -37,6 +36,7 @@ use crate::dispatch::{DispatchInner, DispatchService};
 use crate::gvl::{self, Cancel, DispatchContext};
 use crate::log::{Level, LogEvent, LogSink};
 use crate::net::{LimitedIncoming, bind, listen, listen_inherited};
+use crate::sync::Mutex;
 
 /// The overall-server health entry (empty service name).
 const OVERALL: &str = "";

data/ext/tonic_rails_native/src/sync.rs

@@ -0,0 +1,42 @@
+//! `Mutex` and `Condvar` over `std::sync`, with parking_lot's API: `lock`
+//! returns the guard, and a panic while holding a lock doesn't poison it.
+//!
+//! Why not parking_lot: under contention it spins and then calls `Sleep`,
+//! and on Windows (mingw) the extension's `Sleep` import binds to the one
+//! Ruby's DLL exports.  Ruby's `Sleep` expects a Ruby thread, so on a tokio
+//! thread it crashed the process (#41).  std's locks use SRW locks and
+//! never call `Sleep`.
+
+use std::sync::{MutexGuard, PoisonError};
+
+#[derive(Default)]
+pub struct Mutex<T>(std::sync::Mutex<T>);
+
+impl<T> Mutex<T> {
+    pub const fn new(value: T) -> Self {
+        Self(std::sync::Mutex::new(value))
+    }
+
+    pub fn lock(&self) -> MutexGuard<'_, T> {
+        self.0.lock().unwrap_or_else(PoisonError::into_inner)
+    }
+
+    pub fn get_mut(&mut self) -> &mut T {
+        self.0.get_mut().unwrap_or_else(PoisonError::into_inner)
+    }
+}
+
+#[derive(Default)]
+pub struct Condvar(std::sync::Condvar);
+
+impl Condvar {
+    /// Block until notified, releasing `guard` meanwhile.  Wakes can be
+    /// spurious: wait in a loop on the condition.
+    pub fn wait<'a, T>(&self, guard: MutexGuard<'a, T>) -> MutexGuard<'a, T> {
+        self.0.wait(guard).unwrap_or_else(PoisonError::into_inner)
+    }
+
+    pub fn notify_one(&self) {
+        self.0.notify_one();
+    }
+}

data/lib/tonic/rails/client/retry_policy.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Tonic
+  module Rails
+    class Client
+      # When and how often a call is retried, following gRPC's retry design
+      # (A6).  Pass one, a Hash of its fields or `true` for the defaults as
+      # `retries:` to Client.new or to a single call.
+      #
+      #   Tonic::Rails::Client.new("items.internal:50051", retries: true)
+      #   Tonic::Rails::Client.new("items.internal:50051", retries: { max_attempts: 4 })
+      #   items.get_item({ id: 1 }, retries: false)
+      #
+      # Only calls the server hasn't started on are retried: the connection
+      # failed, or the server answered with a status and no response
+      # headers.  Each wait is random, up to a cap that grows from
+      # `initial_backoff` by `multiplier` per attempt to `max_backoff`.
+      # Retrying stops when the next attempt couldn't start before the
+      # call's deadline.
+      RetryPolicy = Data.define(:max_attempts, :codes, :initial_backoff, :max_backoff, :multiplier) do
+        # Defaults for the fields left out.
+        def self.defaults
+          { max_attempts: 3, codes: [:unavailable], initial_backoff: 0.1, max_backoff: 1, multiplier: 2 }
+        end
+
+        # @param option [RetryPolicy, Hash, true, false, nil]
+        # @return [RetryPolicy, nil] nil for no retries
+        def self.from(option)
+          case option
+          when nil, false then nil
+          when true then new(**defaults)
+          when Hash then new(**defaults, **option)
+          when self then option
+          else raise ArgumentError, "retries must be true, false, a Hash or a #{name}, got #{option.class}"
+          end
+        end
+
+        # @param max_attempts [Integer] attempts in all, from 2 to 5.  gRPC
+        #   caps it at 5, so a misconfigured client can't multiply an
+        #   outage's traffic by more than that.
+        # @param codes [Array<Symbol, Integer>] status codes worth retrying
+        # @param initial_backoff [Numeric] seconds; the cap on the first wait
+        # @param max_backoff [Numeric] seconds; the cap on any wait
+        # @param multiplier [Numeric] how much the cap grows per attempt
+        def initialize(max_attempts:, codes:, initial_backoff:, max_backoff:, multiplier:)
+          unless max_attempts.is_a?(Integer) && max_attempts.between?(2, 5)
+            raise ArgumentError, "max_attempts must be an Integer from 2 to 5"
+          end
+
+          codes = Array(codes).map { |code| RpcError.normalize_code(code) }.uniq.freeze
+          raise ArgumentError, "codes must name at least one status code other than OK" if codes.empty? || codes.include?(0)
+
+          [[:initial_backoff, initial_backoff], [:max_backoff, max_backoff]].each do |field, value|
+            raise ArgumentError, "#{field} must be a positive number of seconds" unless value.is_a?(Numeric) && value.positive?
+          end
+          raise ArgumentError, "multiplier must be at least 1" unless multiplier.is_a?(Numeric) && multiplier >= 1
+
+          super
+        end
+
+        # @param code [Integer]
+        def retry?(code)
+          codes.include?(code)
+        end
+
+        # Seconds to wait after failed attempt number `attempt` (from 1).
+        def backoff(attempt)
+          rand * [initial_backoff * (multiplier**(attempt - 1)), max_backoff].min
+        end
+      end
+    end
+  end
+end

data/lib/tonic/rails/client.rb

@@ -14,6 +14,10 @@ module Tonic
     #   item = items.get_item({ id: 1 }, deadline: 0.5, metadata: { "x-request-id" => request_id })
     #   items.list_items(category: "books").each { |item| ... }
     #
+    # Retries are opt-in: `retries: true` (or a Hash; see RetryPolicy) on
+    # the client or a call retries calls the server never started on.
+    # `method_retries:` sets them per service or method.
+    #
     # Stubs are built from the protobuf descriptors that
     # `tonic_rails:compile_protos` (protoc --ruby_out) generates; no
     # `_services_pb.rb` is needed.  A failed call raises
@@ -29,12 +33,24 @@ module Tonic
 
       DEFAULT_MAX_MESSAGE_SIZE = 4 * 1024 * 1024
 
+      # Trailer a server sends to ask for a retry after so many
+      # milliseconds, or (with a negative value) for none.
+      RETRY_PUSHBACK = "grpc-retry-pushback-ms"
+      private_constant :RETRY_PUSHBACK
+
       # @return [String] "host:port"
       attr_reader :address
 
       # @return [Numeric, nil] default deadline in seconds
       attr_reader :deadline
 
+      # @return [RetryPolicy, nil] how calls are retried, unless they say otherwise
+      attr_reader :retries
+
+      # @return [Hash{String=>RetryPolicy, nil}] retry policies by service or
+      #   method name, which take precedence over #retries
+      attr_reader :method_retries
+
       # @param address [String] "host:port"
       # @param tls [Boolean] false for plaintext, which must be explicit
       # @param tls_ca_path [String, nil] CA certificates (PEM) to trust instead
@@ -48,9 +64,17 @@ module Tonic
       # @param metadata [Hash] sent with every call (see Tonic::Rails::Metadata)
       # @param max_message_size [Integer] largest message sent or received, in bytes
       # @param connect_timeout [Numeric] seconds to wait for a connection
+      # @param retries [RetryPolicy, Hash, Boolean] retry calls the server
+      #   never started on (see RetryPolicy); off by default
+      # @param method_retries [Hash{String=>RetryPolicy, Hash, Boolean}]
+      #   retries for some services ("demo.v1.ItemService") or methods
+      #   ("demo.v1.ItemService/GetItem"), instead of `retries`.  A method's
+      #   entry beats its service's.
+      # @param logger [Logger, nil] where retries are logged; defaults to
+      #   Tonic::Rails.configuration.logger
       def initialize(address, tls: true, tls_ca_path: nil, tls_cert_path: nil, tls_key_path: nil,
                      tls_server_name: nil, deadline: nil, metadata: {}, max_message_size: DEFAULT_MAX_MESSAGE_SIZE,
-                     connect_timeout: 5)
+                     connect_timeout: 5, retries: false, method_retries: {}, logger: nil)
         Tonic::Rails.native_extension!
         validate_address!(address)
         validate_tls!(tls, tls_cert_path, tls_key_path)
@@ -62,6 +86,9 @@ module Tonic
 
         @address = address
         @deadline = deadline
+        @retries = RetryPolicy.from(retries)
+        @method_retries = method_retry_policies(method_retries)
+        @logger = logger
         @metadata = Metadata.encode(metadata)
         @stubs = {}
         tls_options = [read_pem(tls_ca_path, "CA"), read_pem(tls_cert_path, "certificate"),
@@ -106,14 +133,16 @@ module Tonic
 
       # Make a unary call.  Stubs call this; use them instead.
       # @api private
-      def unary(path, input, output, request, deadline: nil, metadata: {}, return_call: false)
-        message = request_message(path, input, request)
+      def unary(path, input, output, request, deadline: nil, metadata: {}, return_call: false, retries: nil)
+        bytes = input.encode(request_message(path, input, request))
         metadata = call_metadata(metadata)
-        timeout_ms = remaining_ms(deadline_at(deadline))
+        deadline_at = deadline_at(deadline)
+        policy = retry_policy(path, retries)
 
-        Instrumentation.instrument_request(path, address, metadata) do
-          code, error_message, response, headers, trailers, details =
-            @native.unary(path, input.encode(message), metadata, timeout_ms)
+        Instrumentation.instrument_request(path, address, metadata) do |payload|
+          code, error_message, response, headers, trailers, details = with_retries(path, policy, deadline_at, payload) do
+            @native.unary(path, bytes, metadata, remaining_ms(deadline_at))
+          end
           headers = decode_metadata(headers)
           trailers = decode_metadata(trailers)
           check_status!(code, error_message, trailers, details)
@@ -125,28 +154,25 @@ module Tonic
 
       # Start a server-streaming call.  Stubs call this; use them instead.
       # @api private
-      def server_streaming(path, input, output, request, deadline: nil, metadata: {})
-        message = request_message(path, input, request)
+      def server_streaming(path, input, output, request, deadline: nil, metadata: {}, retries: nil)
+        bytes = input.encode(request_message(path, input, request))
         metadata = call_metadata(metadata)
         # Fixed now, so the stream keeps the deadline of the RPC this
         # thread is handling even if it's read later.
         deadline_at = deadline_at(deadline)
+        policy = retry_policy(path, retries)
 
         ServerStream.new(path) do |stream, block|
-          timeout_ms = remaining_ms(deadline_at)
-          Instrumentation.instrument_request(path, address, metadata) do
-            native = @native.server_streaming(path, input.encode(message), metadata, timeout_ms)
-            begin
-              while (bytes = native.next)
-                block.call(decode_response(output, bytes, path))
-              end
-              code, error_message, _, headers, trailers, details = native.result
-              trailers = decode_metadata(trailers)
-              stream.finish(decode_metadata(headers), trailers)
-              check_status!(code, error_message, trailers, details)
-            ensure
-              native.close
+          Instrumentation.instrument_request(path, address, metadata) do |payload|
+            # A message arrives only after the response headers, which
+            # commit the call, so an attempt that yielded anything is never
+            # retried.
+            code, error_message, _, headers, trailers, details = with_retries(path, policy, deadline_at, payload) do
+              read_stream(@native.server_streaming(path, bytes, metadata, remaining_ms(deadline_at)), output, path, &block)
             end
+            trailers = decode_metadata(trailers)
+            stream.finish(decode_metadata(headers), trailers)
+            check_status!(code, error_message, trailers, details)
           end
         end
       end
@@ -222,6 +248,102 @@ module Tonic
         (remaining * 1000).ceil
       end
 
+      METHOD_NAME = %r{\A[A-Za-z_][\w.]*(/[A-Za-z_]\w*)?\z}
+      private_constant :METHOD_NAME
+
+      def method_retry_policies(option)
+        raise ArgumentError, "method_retries must be a Hash" unless option.is_a?(Hash)
+
+        option.to_h do |name, retries|
+          name = name.to_s.delete_prefix("/")
+          unless name.match?(METHOD_NAME)
+            raise ArgumentError, "method_retries keys must be \"package.Service\" or \"package.Service/Method\", " \
+                                 "got #{name.inspect}"
+          end
+
+          [name, RetryPolicy.from(retries)]
+        end.freeze
+      end
+
+      # The most specific policy for a call: its own, its method's, its
+      # service's, then the client's.
+      def retry_policy(path, retries)
+        return RetryPolicy.from(retries) unless retries.nil?
+        return @retries if @method_retries.empty?
+
+        method = path.delete_prefix("/")
+        service = method.split("/", 2).first
+        return @method_retries[method] if @method_retries.key?(method)
+        return @method_retries[service] if @method_retries.key?(service)
+
+        @retries
+      end
+
+      # Yield each message of a native stream, then return its result.
+      def read_stream(native, output, path)
+        while (bytes = native.next)
+          yield decode_response(output, bytes, path)
+        end
+        native.result
+      ensure
+        native.close
+      end
+
+      # Run attempts of a call (the block returns a native result) until one
+      # succeeds or shouldn't be retried; return its result.
+      def with_retries(path, policy, deadline_at, payload)
+        attempt = 1
+        loop do
+          payload[:attempts] = attempt if payload
+          result = yield
+          delay = retry_delay(policy, attempt, result, deadline_at)
+          if delay.nil?
+            log_gave_up(path, attempt, result) if attempt > 1 && !result[0].zero?
+            return result
+          end
+
+          log_retry(path, attempt, policy.max_attempts, result, delay)
+          sleep(delay)
+          attempt += 1
+        end
+      end
+
+      # Seconds to wait before retrying, or nil not to retry.  The server
+      # can ask for a delay, or for no retry, with grpc-retry-pushback-ms.
+      def retry_delay(policy, attempt, result, deadline_at)
+        code, _, _, _, trailers, _, committed = result
+        return if policy.nil? || code.zero? || committed || attempt >= policy.max_attempts || !policy.retry?(code)
+
+        pushback = trailers.assoc(RETRY_PUSHBACK)&.last
+        delay = if pushback.nil? then policy.backoff(attempt)
+        elsif pushback.match?(/\A\d+\z/) then pushback.to_i / 1000.0
+        end
+        return if delay.nil?
+        return if deadline_at && Process.clock_gettime(Process::CLOCK_MONOTONIC) + delay >= deadline_at
+
+        delay
+      end
+
+      def logger
+        @logger || Tonic::Rails.configuration.logger
+      end
+
+      def log_retry(path, attempt, max_attempts, result, delay)
+        logger&.warn(format("tonic-rails: %<path>s failed with %<status>s (attempt %<attempt>d of %<max>d); " \
+                            "retrying in %<ms>dms", path: path, status: status_text(result), attempt: attempt,
+                                                    max: max_attempts, ms: (delay * 1000).round))
+      end
+
+      def log_gave_up(path, attempts, result)
+        logger&.warn("tonic-rails: #{path} failed with #{status_text(result)} after #{attempts} attempts")
+      end
+
+      def status_text(result)
+        code, message = result
+        text = RpcError::CODE_NAMES.fetch(code, code).to_s.upcase
+        message.empty? ? text : "#{text}: #{message}"
+      end
+
       def check_status!(code, message, trailers, details)
         return if code.zero?
 
@@ -282,16 +404,18 @@ module Tonic
             input = method.input_type.msgclass
             output = method.output_type.msgclass
             if method.server_streaming
-              define_method(name) do |request = nil, deadline: nil, metadata: {}, **fields|
+              define_method(name) do |request = nil, deadline: nil, metadata: {}, retries: nil, **fields|
                 request = rpc_request(name, request, fields)
-                @client.server_streaming(path, input, output, request, deadline: deadline, metadata: metadata)
+                @client.server_streaming(path, input, output, request, deadline: deadline, metadata: metadata,
+                                                                       retries: retries)
               end
             else
               # `get_item(id: 1)` works as well as `get_item({ id: 1 })`.
-              define_method(name) do |request = nil, deadline: nil, metadata: {}, return_call: false, **fields|
+              define_method(name) do |request = nil, deadline: nil, metadata: {}, return_call: false, retries: nil,
+                                      **fields|
                 request = rpc_request(name, request, fields)
                 @client.unary(path, input, output, request, deadline: deadline, metadata: metadata,
-                                                            return_call: return_call)
+                                                            return_call: return_call, retries: retries)
               end
             end
           end
@@ -369,3 +493,5 @@ module Tonic
     end
   end
 end
+
+require "tonic/rails/client/retry_policy"

data/lib/tonic/rails/instrumentation.rb

@@ -25,8 +25,8 @@ module Tonic
       EVENT = "process_rpc.tonic_rails"
 
       # Outgoing calls made with Tonic::Rails::Client publish
-      # "request_rpc.tonic_rails", with the same payload plus :address,
-      # the server called.
+      # "request_rpc.tonic_rails", with the same payload plus :address, the
+      # server called, and :attempts, how many times it was tried.
       REQUEST_EVENT = "request_rpc.tonic_rails"
 
       module_function
@@ -41,16 +41,17 @@ module Tonic
         publish(REQUEST_EVENT, path, metadata, address: address, &)
       end
 
+      # Yields the payload, or nil when nothing is listening.
       # @api private
       def publish(event, path, metadata, **extra)
         # Without subscribers, skip building the payload: this runs on
         # every RPC.
-        return yield unless ActiveSupport::Notifications.notifier.listening?(event)
+        return yield(nil) unless ActiveSupport::Notifications.notifier.listening?(event)
 
         service, rpc = path.delete_prefix("/").split("/", 2)
         payload = { path: path, service: service, rpc: rpc, metadata: metadata, code: nil, status: nil, **extra }
         ActiveSupport::Notifications.instrument(event, payload) do
-          result = yield
+          result = yield(payload)
           payload[:code] = 0
           payload[:status] = :ok
           result

data/lib/tonic/rails/version.rb

@@ -2,6 +2,6 @@
 
 module Tonic
   module Rails
-    VERSION = "0.6.0"
+    VERSION = "0.8.0"
   end
 end

data/lib/tonic-rails.rb

@@ -81,11 +81,13 @@ module Tonic
         # Precompiled platform gems ship one binary per Ruby minor version.
         ruby_version = RUBY_VERSION[/\d+\.\d+/]
         require "tonic/rails/#{ruby_version}/tonic_rails_native"
-      rescue LoadError
+      rescue LoadError => versioned
         begin
           require "tonic/rails/tonic_rails_native"
         rescue LoadError => e
-          @native_load_error = e
+          # A precompiled binary that exists but won't load says more than
+          # the missing fallback does.
+          @native_load_error = versioned.message.include?("cannot load such file") ? e : versioned
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tonic-rails
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Matt Tanous
@@ -81,6 +81,7 @@ files:
 - ext/tonic_rails_native/src/log.rs
 - ext/tonic_rails_native/src/net.rs
 - ext/tonic_rails_native/src/server.rs
+- ext/tonic_rails_native/src/sync.rs
 - lib/generators/tonic_rails/install/install_generator.rb
 - lib/generators/tonic_rails/install/templates/tonic_rails.rb.tt
 - lib/generators/tonic_rails/service/service_generator.rb
@@ -92,6 +93,7 @@ files:
 - lib/tonic-rails.rb
 - lib/tonic/rails/base.rb
 - lib/tonic/rails/client.rb
+- lib/tonic/rails/client/retry_policy.rb
 - lib/tonic/rails/cluster.rb
 - lib/tonic/rails/configuration.rb
 - lib/tonic/rails/health.rb