grpc 1.64.3 → 1.65.0

data/src/core/lib/transport/connectivity_state.cc

@@ -29,8 +29,6 @@
 
 namespace grpc_core {
 
-TraceFlag grpc_connectivity_state_trace(false, "connectivity_state");
-
 const char* ConnectivityStateName(grpc_connectivity_state state) {
   switch (state) {
     case GRPC_CHANNEL_IDLE:
@@ -73,7 +71,7 @@ class AsyncConnectivityStateWatcherInterface::Notifier {
  private:
   static void SendNotification(void* arg, grpc_error_handle /*ignored*/) {
     Notifier* self = static_cast<Notifier*>(arg);
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_connectivity_state_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(connectivity_state)) {
       gpr_log(GPR_INFO, "watcher %p: delivering async notification for %s (%s)",
               self->watcher_.get(), ConnectivityStateName(self->state_),
               self->status_.ToString().c_str());
@@ -104,7 +102,7 @@ ConnectivityStateTracker::~ConnectivityStateTracker() {
       state_.load(std::memory_order_relaxed);
   if (current_state == GRPC_CHANNEL_SHUTDOWN) return;
   for (const auto& p : watchers_) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_connectivity_state_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(connectivity_state)) {
       gpr_log(GPR_INFO,
               "ConnectivityStateTracker %s[%p]: notifying watcher %p: %s -> %s",
               name_, this, p.first, ConnectivityStateName(current_state),
@@ -117,14 +115,14 @@ ConnectivityStateTracker::~ConnectivityStateTracker() {
 void ConnectivityStateTracker::AddWatcher(
     grpc_connectivity_state initial_state,
     OrphanablePtr<ConnectivityStateWatcherInterface> watcher) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_connectivity_state_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(connectivity_state)) {
     gpr_log(GPR_INFO, "ConnectivityStateTracker %s[%p]: add watcher %p", name_,
             this, watcher.get());
   }
   grpc_connectivity_state current_state =
       state_.load(std::memory_order_relaxed);
   if (initial_state != current_state) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_connectivity_state_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(connectivity_state)) {
       gpr_log(GPR_INFO,
               "ConnectivityStateTracker %s[%p]: notifying watcher %p: %s -> %s",
               name_, this, watcher.get(), ConnectivityStateName(initial_state),
@@ -141,7 +139,7 @@ void ConnectivityStateTracker::AddWatcher(
 
 void ConnectivityStateTracker::RemoveWatcher(
     ConnectivityStateWatcherInterface* watcher) {
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_connectivity_state_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(connectivity_state)) {
     gpr_log(GPR_INFO, "ConnectivityStateTracker %s[%p]: remove watcher %p",
             name_, this, watcher);
   }
@@ -154,7 +152,7 @@ void ConnectivityStateTracker::SetState(grpc_connectivity_state state,
   grpc_connectivity_state current_state =
       state_.load(std::memory_order_relaxed);
   if (state == current_state) return;
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_connectivity_state_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(connectivity_state)) {
     gpr_log(GPR_INFO, "ConnectivityStateTracker %s[%p]: %s -> %s (%s, %s)",
             name_, this, ConnectivityStateName(current_state),
             ConnectivityStateName(state), reason, status.ToString().c_str());
@@ -162,7 +160,7 @@ void ConnectivityStateTracker::SetState(grpc_connectivity_state state,
   state_.store(state, std::memory_order_relaxed);
   status_ = status;
   for (const auto& p : watchers_) {
-    if (GRPC_TRACE_FLAG_ENABLED(grpc_connectivity_state_trace)) {
+    if (GRPC_TRACE_FLAG_ENABLED(connectivity_state)) {
       gpr_log(GPR_INFO,
               "ConnectivityStateTracker %s[%p]: notifying watcher %p: %s -> %s",
               name_, this, p.first, ConnectivityStateName(current_state),
@@ -177,7 +175,7 @@ void ConnectivityStateTracker::SetState(grpc_connectivity_state state,
 
 grpc_connectivity_state ConnectivityStateTracker::state() const {
   grpc_connectivity_state state = state_.load(std::memory_order_relaxed);
-  if (GRPC_TRACE_FLAG_ENABLED(grpc_connectivity_state_trace)) {
+  if (GRPC_TRACE_FLAG_ENABLED(connectivity_state)) {
     gpr_log(GPR_INFO, "ConnectivityStateTracker %s[%p]: get current state: %s",
             name_, this, ConnectivityStateName(state));
   }

data/src/core/lib/transport/connectivity_state.h

@@ -35,8 +35,6 @@
 
 namespace grpc_core {
 
-extern TraceFlag grpc_connectivity_state_trace;
-
 // Enum to string conversion.
 const char* ConnectivityStateName(grpc_connectivity_state state);
 

data/src/core/lib/transport/interception_chain.cc

@@ -0,0 +1,155 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "src/core/lib/transport/interception_chain.h"
+
+#include <cstddef>
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/gprpp/match.h"
+#include "src/core/lib/transport/call_destination.h"
+#include "src/core/lib/transport/call_filters.h"
+#include "src/core/lib/transport/call_spine.h"
+#include "src/core/lib/transport/metadata.h"
+
+namespace grpc_core {
+
+std::atomic<size_t> InterceptionChainBuilder::next_filter_id_{0};
+
+///////////////////////////////////////////////////////////////////////////////
+// HijackedCall
+
+CallInitiator HijackedCall::MakeCall() {
+  auto metadata = Arena::MakePooled<ClientMetadata>();
+  *metadata = metadata_->Copy();
+  return MakeCallWithMetadata(std::move(metadata));
+}
+
+CallInitiator HijackedCall::MakeCallWithMetadata(
+    ClientMetadataHandle metadata) {
+  auto call = MakeCallPair(std::move(metadata), call_handler_.event_engine(),
+                           call_handler_.arena()->Ref());
+  destination_->StartCall(std::move(call.handler));
+  return std::move(call.initiator);
+}
+
+namespace {
+class CallStarter final : public UnstartedCallDestination {
+ public:
+  CallStarter(RefCountedPtr<CallFilters::Stack> stack,
+              RefCountedPtr<CallDestination> destination)
+      : stack_(std::move(stack)), destination_(std::move(destination)) {}
+
+  void Orphaned() override {
+    stack_.reset();
+    destination_.reset();
+  }
+
+  void StartCall(UnstartedCallHandler unstarted_call_handler) override {
+    destination_->HandleCall(unstarted_call_handler.StartCall(stack_));
+  }
+
+ private:
+  RefCountedPtr<CallFilters::Stack> stack_;
+  RefCountedPtr<CallDestination> destination_;
+};
+
+class TerminalInterceptor final : public UnstartedCallDestination {
+ public:
+  explicit TerminalInterceptor(
+      RefCountedPtr<CallFilters::Stack> stack,
+      RefCountedPtr<UnstartedCallDestination> destination)
+      : stack_(std::move(stack)), destination_(std::move(destination)) {}
+
+  void Orphaned() override {
+    stack_.reset();
+    destination_.reset();
+  }
+
+  void StartCall(UnstartedCallHandler unstarted_call_handler) override {
+    unstarted_call_handler.SpawnGuarded(
+        "start_call",
+        Map(interception_chain_detail::HijackCall(unstarted_call_handler,
+                                                  destination_, stack_),
+            [](ValueOrFailure<HijackedCall> hijacked_call) -> StatusFlag {
+              if (!hijacked_call.ok()) return Failure{};
+              ForwardCall(hijacked_call.value().original_call_handler(),
+                          hijacked_call.value().MakeLastCall());
+              return Success{};
+            }));
+  }
+
+ private:
+  RefCountedPtr<CallFilters::Stack> stack_;
+  RefCountedPtr<UnstartedCallDestination> destination_;
+};
+}  // namespace
+
+///////////////////////////////////////////////////////////////////////////////
+// InterceptionChain::Builder
+
+void InterceptionChainBuilder::AddInterceptor(
+    absl::StatusOr<RefCountedPtr<Interceptor>> interceptor) {
+  if (!status_.ok()) return;
+  if (!interceptor.ok()) {
+    status_ = interceptor.status();
+    return;
+  }
+  (*interceptor)->filter_stack_ = MakeFilterStack();
+  if (top_interceptor_ == nullptr) {
+    top_interceptor_ = std::move(*interceptor);
+  } else {
+    Interceptor* previous = top_interceptor_.get();
+    while (previous->wrapped_destination_ != nullptr) {
+      previous = DownCast<Interceptor*>(previous->wrapped_destination_.get());
+    }
+    previous->wrapped_destination_ = std::move(*interceptor);
+  }
+}
+
+absl::StatusOr<RefCountedPtr<UnstartedCallDestination>>
+InterceptionChainBuilder::Build(FinalDestination final_destination) {
+  if (!status_.ok()) return status_;
+  // Build the final UnstartedCallDestination in the chain - what we do here
+  // depends on both the type of the final destination and the filters we have
+  // that haven't been captured into an Interceptor yet.
+  RefCountedPtr<UnstartedCallDestination> terminator = Match(
+      final_destination,
+      [this](RefCountedPtr<UnstartedCallDestination> final_destination)
+          -> RefCountedPtr<UnstartedCallDestination> {
+        if (stack_builder_.has_value()) {
+          return MakeRefCounted<TerminalInterceptor>(MakeFilterStack(),
+                                                     final_destination);
+        }
+        return final_destination;
+      },
+      [this](RefCountedPtr<CallDestination> final_destination)
+          -> RefCountedPtr<UnstartedCallDestination> {
+        return MakeRefCounted<CallStarter>(MakeFilterStack(),
+                                           std::move(final_destination));
+      });
+  // Now append the terminator to the interceptor chain.
+  if (top_interceptor_ == nullptr) {
+    return std::move(terminator);
+  }
+  Interceptor* previous = top_interceptor_.get();
+  while (previous->wrapped_destination_ != nullptr) {
+    previous = DownCast<Interceptor*>(previous->wrapped_destination_.get());
+  }
+  previous->wrapped_destination_ = std::move(terminator);
+  return std::move(top_interceptor_);
+}
+
+}  // namespace grpc_core

data/src/core/lib/transport/interception_chain.h

@@ -0,0 +1,236 @@
+// Copyright 2024 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_SRC_CORE_LIB_TRANSPORT_INTERCEPTION_CHAIN_H
+#define GRPC_SRC_CORE_LIB_TRANSPORT_INTERCEPTION_CHAIN_H
+
+#include <memory>
+#include <vector>
+
+#include <grpc/support/port_platform.h>
+
+#include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/transport/call_destination.h"
+#include "src/core/lib/transport/call_filters.h"
+#include "src/core/lib/transport/call_spine.h"
+#include "src/core/lib/transport/metadata.h"
+
+namespace grpc_core {
+
+class InterceptionChainBuilder;
+
+// One hijacked call. Using this we can get access to the CallHandler for the
+// call object above us, the processed metadata from any filters/interceptors
+// above us, and also create new CallInterceptor objects that will be handled
+// below.
+class HijackedCall final {
+ public:
+  HijackedCall(ClientMetadataHandle metadata,
+               RefCountedPtr<UnstartedCallDestination> destination,
+               CallHandler call_handler)
+      : metadata_(std::move(metadata)),
+        destination_(std::move(destination)),
+        call_handler_(std::move(call_handler)) {}
+
+  // Create a new call and pass it down the stack.
+  // This can be called as many times as needed.
+  CallInitiator MakeCall();
+  // Per MakeCall(), but precludes creating further calls.
+  // Allows us to optimize by not copying initial metadata.
+  CallInitiator MakeLastCall() {
+    return MakeCallWithMetadata(std::move(metadata_));
+  }
+
+  CallHandler& original_call_handler() { return call_handler_; }
+
+  ClientMetadata& client_metadata() { return *metadata_; }
+
+ private:
+  CallInitiator MakeCallWithMetadata(ClientMetadataHandle metadata);
+
+  ClientMetadataHandle metadata_;
+  RefCountedPtr<UnstartedCallDestination> destination_;
+  CallHandler call_handler_;
+};
+
+namespace interception_chain_detail {
+
+inline auto HijackCall(UnstartedCallHandler unstarted_call_handler,
+                       RefCountedPtr<UnstartedCallDestination> destination,
+                       RefCountedPtr<CallFilters::Stack> stack) {
+  auto call_handler = unstarted_call_handler.StartCall(stack);
+  return Map(
+      call_handler.PullClientInitialMetadata(),
+      [call_handler,
+       destination](ValueOrFailure<ClientMetadataHandle> metadata) mutable
+      -> ValueOrFailure<HijackedCall> {
+        if (!metadata.ok()) return Failure{};
+        return HijackedCall(std::move(metadata.value()), std::move(destination),
+                            std::move(call_handler));
+      });
+}
+
+}  // namespace interception_chain_detail
+
+// A delegating UnstartedCallDestination for use as a hijacking filter.
+// Implementations may look at the unprocessed initial metadata
+// and decide to do one of two things:
+//
+// 1. It can hijack the call. Returns a HijackedCall object that can
+//    be used to start new calls with the same metadata.
+//
+// 2. It can consume the call by calling `Consume`.
+//
+// Upon the StartCall call the UnstartedCallHandler will be from the last
+// *Interceptor* in the call chain (without having been processed by any
+// intervening filters) -- note that this is commonly not useful (not enough
+// guarantees), and so it's usually better to Hijack and examine the metadata.
+class Interceptor : public UnstartedCallDestination {
+ protected:
+  // Returns a promise that resolves to a HijackedCall instance.
+  // Hijacking is the process of taking over a call and starting one or more new
+  // ones.
+  auto Hijack(UnstartedCallHandler unstarted_call_handler) {
+    return interception_chain_detail::HijackCall(
+        std::move(unstarted_call_handler), wrapped_destination_, filter_stack_);
+  }
+
+  // Consume this call - it will not be passed on to any further filters.
+  CallHandler Consume(UnstartedCallHandler unstarted_call_handler) {
+    return unstarted_call_handler.StartCall(filter_stack_);
+  }
+
+  // TODO(ctiller): Consider a Passthrough() method that allows the call to be
+  // passed on to the next filter in the chain without any interception by the
+  // current filter.
+
+ private:
+  friend class InterceptionChainBuilder;
+
+  RefCountedPtr<UnstartedCallDestination> wrapped_destination_;
+  RefCountedPtr<CallFilters::Stack> filter_stack_;
+};
+
+class InterceptionChainBuilder final {
+ public:
+  // The kind of destination that the chain will eventually call.
+  // We can bottom out in various types depending on where we're intercepting:
+  // - The top half of the client channel wants to terminate on a
+  //   UnstartedCallDestination (specifically the LB call destination).
+  // - The bottom half of the client channel and the server code wants to
+  //   terminate on a ClientTransport - which unlike a
+  //   UnstartedCallDestination demands a started CallHandler.
+  // There's some adaption code that's needed to start filters just prior
+  // to the bottoming out, and some design considerations to make with that.
+  // One way (that's not chosen here) would be to have the caller of the
+  // Builder provide something that can build an adaptor
+  // UnstartedCallDestination with parameters supplied by this builder - that
+  // disperses the responsibility of building the adaptor to the caller, which
+  // is not ideal - we might want to adjust the way this construct is built in
+  // the future, and building is a builder responsibility.
+  // Instead, we declare a relatively closed set of destinations here, and
+  // hide the adaptors inside the builder at build time.
+  using FinalDestination =
+      absl::variant<RefCountedPtr<UnstartedCallDestination>,
+                    RefCountedPtr<CallDestination>>;
+
+  explicit InterceptionChainBuilder(ChannelArgs args)
+      : args_(std::move(args)) {}
+
+  // Add a filter with a `Call` class as an inner member.
+  // Call class must be one compatible with the filters described in
+  // call_filters.h.
+  template <typename T>
+  absl::enable_if_t<sizeof(typename T::Call) != 0, InterceptionChainBuilder&>
+  Add() {
+    if (!status_.ok()) return *this;
+    auto filter = T::Create(args_, {FilterInstanceId(FilterTypeId<T>())});
+    if (!filter.ok()) {
+      status_ = filter.status();
+      return *this;
+    }
+    auto& sb = stack_builder();
+    sb.Add(filter.value().get());
+    sb.AddOwnedObject(std::move(filter.value()));
+    return *this;
+  };
+
+  // Add a filter that is an interceptor - one that can hijack calls.
+  template <typename T>
+  absl::enable_if_t<std::is_base_of<Interceptor, T>::value,
+                    InterceptionChainBuilder&>
+  Add() {
+    AddInterceptor(T::Create(args_, {FilterInstanceId(FilterTypeId<T>())}));
+    return *this;
+  };
+
+  // Add a filter that just mutates client initial metadata.
+  template <typename F>
+  void AddOnClientInitialMetadata(F f) {
+    stack_builder().AddOnClientInitialMetadata(std::move(f));
+  }
+
+  // Add a filter that just mutates server trailing metadata.
+  template <typename F>
+  void AddOnServerTrailingMetadata(F f) {
+    stack_builder().AddOnServerTrailingMetadata(std::move(f));
+  }
+
+  void Fail(absl::Status status) {
+    CHECK(!status.ok()) << status;
+    if (status_.ok()) status_ = std::move(status);
+  }
+
+  // Build this stack
+  absl::StatusOr<RefCountedPtr<UnstartedCallDestination>> Build(
+      FinalDestination final_destination);
+
+  const ChannelArgs& channel_args() const { return args_; }
+
+ private:
+  CallFilters::StackBuilder& stack_builder() {
+    if (!stack_builder_.has_value()) stack_builder_.emplace();
+    return *stack_builder_;
+  }
+
+  RefCountedPtr<CallFilters::Stack> MakeFilterStack() {
+    auto stack = stack_builder().Build();
+    stack_builder_.reset();
+    return stack;
+  }
+
+  template <typename T>
+  static size_t FilterTypeId() {
+    static const size_t id =
+        next_filter_id_.fetch_add(1, std::memory_order_relaxed);
+    return id;
+  }
+
+  size_t FilterInstanceId(size_t filter_type) {
+    return filter_type_counts_[filter_type]++;
+  }
+
+  void AddInterceptor(absl::StatusOr<RefCountedPtr<Interceptor>> interceptor);
+
+  ChannelArgs args_;
+  absl::optional<CallFilters::StackBuilder> stack_builder_;
+  RefCountedPtr<Interceptor> top_interceptor_;
+  absl::Status status_;
+  std::map<size_t, size_t> filter_type_counts_;
+  static std::atomic<size_t> next_filter_id_;
+};
+
+}  // namespace grpc_core
+
+#endif  // GRPC_SRC_CORE_LIB_TRANSPORT_INTERCEPTION_CHAIN_H

data/src/core/lib/transport/metadata_batch.h

@@ -528,6 +528,14 @@ struct WaitForReady {
   static std::string DisplayValue(ValueType x);
 };
 
+// Annotation added by retry code to indicate a transparent retry.
+struct IsTransparentRetry {
+  static absl::string_view DebugKey() { return "IsTransparentRetry"; }
+  static constexpr bool kRepeatable = false;
+  using ValueType = bool;
+  static std::string DisplayValue(ValueType x) { return x ? "true" : "false"; }
+};
+
 // Annotation added by a transport to note that server trailing metadata
 // is a Trailers-Only response.
 struct GrpcTrailersOnly {
@@ -1536,7 +1544,8 @@ using grpc_metadata_batch_base = grpc_core::MetadataMap<
     grpc_core::GrpcStreamNetworkState, grpc_core::PeerString,
     grpc_core::GrpcStatusContext, grpc_core::GrpcStatusFromWire,
     grpc_core::GrpcCallWasCancelled, grpc_core::WaitForReady,
-    grpc_core::GrpcTrailersOnly, grpc_core::GrpcTarPit,
+    grpc_core::IsTransparentRetry, grpc_core::GrpcTrailersOnly,
+    grpc_core::GrpcTarPit,
     grpc_core::GrpcRegisteredMethod GRPC_CUSTOM_CLIENT_METADATA
         GRPC_CUSTOM_SERVER_METADATA>;
 

data/src/core/lib/transport/metadata_info.h

@@ -18,9 +18,9 @@
 #include <grpc/support/port_platform.h>
 
 #include "src/core/ext/transport/chttp2/transport/hpack_constants.h"
-#include "src/core/lib/channel/call_tracer.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/transport/metadata_batch.h"
+#include "src/core/telemetry/call_tracer.h"
 
 namespace grpc_core {
 

data/src/core/lib/transport/transport.cc

@@ -40,9 +40,6 @@
 #include "src/core/lib/slice/slice.h"
 #include "src/core/lib/transport/error_utils.h"
 
-grpc_core::DebugOnlyTraceFlag grpc_trace_stream_refcount(false,
-                                                         "stream_refcount");
-
 void grpc_stream_destroy(grpc_stream_refcount* refcount) {
   if ((grpc_core::ExecCtx::Get()->flags() &
        GRPC_EXEC_CTX_FLAG_THREAD_RESOURCE_LOOP)) {
@@ -81,8 +78,8 @@ void grpc_stream_ref_init(grpc_stream_refcount* refcount, int /*initial_refs*/,
   GRPC_CLOSURE_INIT(&refcount->destroy, cb, cb_arg, grpc_schedule_on_exec_ctx);
 
   new (&refcount->refs) grpc_core::RefCount(
-      1, GRPC_TRACE_FLAG_ENABLED(grpc_trace_stream_refcount) ? "stream_refcount"
-                                                             : nullptr);
+      1,
+      GRPC_TRACE_FLAG_ENABLED(stream_refcount) ? "stream_refcount" : nullptr);
 }
 
 namespace grpc_core {
@@ -192,7 +189,7 @@ struct made_transport_stream_op {
   grpc_closure outer_on_complete;
   grpc_closure* inner_on_complete = nullptr;
   grpc_transport_stream_op_batch op;
-  grpc_transport_stream_op_batch_payload payload{nullptr};
+  grpc_transport_stream_op_batch_payload payload;
 };
 static void destroy_made_transport_stream_op(void* arg,
                                              grpc_error_handle error) {