grpc 1.81.1 → 1.83.0

data/src/core/filter/composite/composite_filter.cc

@@ -102,10 +102,17 @@ CompositeFilter::CompositeFilter(const ChannelArgs& args,
     if (action.type() != ExecuteFilterAction::Type()) return;
     const auto& execute_filter_action =
         DownCast<const ExecuteFilterAction&>(action);
-    InterceptionChainBuilder builder(args, filter_args.blackboard());
+    auto it = config_->merged_config_map.find(&action);
+    GRPC_CHECK(it != config_->merged_config_map.end());
+    auto& merged_configs = it->second;
+    GRPC_CHECK_EQ(merged_configs.size(),
+                  execute_filter_action.filter_chain().size());
+    InterceptionChainBuilder builder(args);
     InterceptionChainBuilderWrapper builder_wrapper(builder);
-    for (const auto& [filter_impl, filter_config] :
-         execute_filter_action.filter_chain()) {
+    for (size_t i = 0; i < merged_configs.size(); ++i) {
+      const auto* filter_impl =
+          execute_filter_action.filter_chain()[i].filter_impl;
+      const auto& filter_config = merged_configs[i];
       // TODO(roth): Currently, this code assumes that it is always
       // running on the client side, so we'll need to remove this
       // assumption in order to make this filter work on the server

data/src/core/filter/composite/composite_filter.h

@@ -116,7 +116,17 @@ class CompositeFilter final : public V3InterceptorToV2Bridge<CompositeFilter> {
       return matcher->ToString();
     }
 
-    std::unique_ptr<XdsMatcher> matcher;
+    std::shared_ptr<const XdsMatcher> matcher;
+
+    // For each action, contains the list of merged filter configs to
+    // use.  This is basically the same as the filter configs in the
+    // matcher actions, except that the configs may include fields set
+    // from the blackboard.
+    // Note: This is not actually part of the identity of the config
+    // (i.e., does not matter for Equals()).
+    absl::flat_hash_map<const XdsMatcher::Action*,
+                        std::vector<RefCountedPtr<const FilterConfig>>>
+        merged_config_map;
   };
 
   static const grpc_channel_filter kFilterVtable;

data/src/core/filter/filter_args.h

@@ -17,7 +17,6 @@
 
 #include <memory>
 
-#include "src/core/filter/blackboard.h"
 #include "src/core/lib/channel/channel_fwd.h"
 #include "src/core/util/match.h"
 #include "src/core/util/ref_counted.h"
@@ -58,12 +57,10 @@ class FilterArgs {
              grpc_channel_element* channel_element,
              size_t (*channel_stack_filter_instance_number)(
                  grpc_channel_stack*, grpc_channel_element*),
-             RefCountedPtr<const FilterConfig> config = nullptr,
-             const Blackboard* blackboard = nullptr)
+             RefCountedPtr<const FilterConfig> config = nullptr)
       : impl_(ChannelStackBased{channel_stack, channel_element,
                                 channel_stack_filter_instance_number}),
-        config_(std::move(config)),
-        blackboard_(blackboard) {}
+        config_(std::move(config)) {}
   // While we're moving to call-v3 we need to have access to
   // grpc_channel_stack & friends here. That means that we can't rely on this
   // type signature from interception_chain.h, which means that we need a way
@@ -71,11 +68,8 @@ class FilterArgs {
   // TODO(ctiller): remove this once we're fully on call-v3
   // NOLINTNEXTLINE(google-explicit-constructor)
   FilterArgs(size_t instance_id,
-             RefCountedPtr<const FilterConfig> config = nullptr,
-             const Blackboard* blackboard = nullptr)
-      : impl_(V3Based{instance_id}),
-        config_(std::move(config)),
-        blackboard_(blackboard) {}
+             RefCountedPtr<const FilterConfig> config = nullptr)
+      : impl_(V3Based{instance_id}), config_(std::move(config)) {}
 
   ABSL_DEPRECATED("Direct access to channel stack is deprecated")
   grpc_channel_stack* channel_stack() const {
@@ -103,17 +97,6 @@ class FilterArgs {
 
   RefCountedPtr<const FilterConfig> config() const { return config_; }
 
-  // Gets the filter state associated with a particular type and key.
-  template <typename T>
-  RefCountedPtr<T> GetState(const std::string& key) const {
-    if (blackboard_ == nullptr) return nullptr;
-    return blackboard_->Get<T>(key);
-  }
-
-  // Do not use unless you know what you're doing -- prefer to use
-  // GetState() instead!
-  const Blackboard* blackboard() const { return blackboard_; }
-
  private:
   friend class ChannelFilter;
 
@@ -132,7 +115,6 @@ class FilterArgs {
   Impl impl_;
 
   const RefCountedPtr<const FilterConfig> config_;
-  const Blackboard* blackboard_ = nullptr;
 };
 
 }  // namespace grpc_core

data/src/core/handshaker/handshaker.cc

@@ -139,9 +139,14 @@ void HandshakeManager::Shutdown(absl::Status error) {
     is_shutdown_ = true;
     // Shutdown the handshaker that's currently in progress, if any.
     if (index_ > 0) {
+      auto& handshaker = *handshakers_[index_ - 1];
       GRPC_CHANNELZ_LOG(args_.trace_node)
-          << "Shutting down handshaker at index " << (index_ - 1);
-      handshakers_[index_ - 1]->Shutdown(std::move(error));
+          << "Shutting down handshaker " << handshaker.name() << " at index "
+          << (index_ - 1);
+      if (IsVerboseChannelzConnectionLoggingEnabled()) {
+        args_.trace_node.Commit();
+      }
+      handshaker.Shutdown(std::move(error));
     }
   }
 }
@@ -187,6 +192,9 @@ void HandshakeManager::CallNextHandshakerLocked(absl::Status error) {
   auto handshaker = handshakers_[index_];
   GRPC_CHANNELZ_LOG(args_.trace_node)
       << " calling handshaker " << handshaker->name() << " at index " << index_;
+  if (IsVerboseChannelzConnectionLoggingEnabled()) {
+    args_.trace_node.Commit();
+  }
   ++index_;
   handshaker->DoHandshake(&args_, [self = Ref()](absl::Status error) mutable {
     MutexLock lock(&self->mu_);

data/src/core/handshaker/security/secure_endpoint.cc

@@ -39,6 +39,8 @@
 #include <vector>
 
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/event_engine/extensions/receive_coalescing_extension.h"
+#include "src/core/lib/event_engine/query_extensions.h"
 #include "src/core/lib/experiments/experiments.h"
 #include "src/core/lib/iomgr/closure.h"
 #include "src/core/lib/iomgr/endpoint.h"
@@ -575,6 +577,17 @@ class FrameProtector : public RefCounted<FrameProtector> {
     }
   }
 
+  // Resets the read staging buffer.
+  // Per the contract of EnableRpcReceiveCoalescing(), this is only
+  // called when there are no outstanding Read() operations on the endpoint.
+  // Therefore, any valid plaintext has already been fully delivered, and
+  // read_staging_buffer_ contains only unused, leftover memory capacity.
+  void ResetReadStagingBuffer() {
+    MutexLock lock(&read_mu_);
+    CSliceUnref(read_staging_buffer_);
+    read_staging_buffer_ = grpc_empty_slice();
+  }
+
  private:
   struct tsi_frame_protector* const protector_;
   struct tsi_zero_copy_grpc_protector* const zero_copy_protector_;
@@ -821,7 +834,8 @@ static const grpc_endpoint_vtable vtable = {endpoint_read,
 namespace grpc_event_engine::experimental {
 namespace {
 
-class SecureEndpoint final : public EventEngine::Endpoint {
+class SecureEndpoint final : public EventEngine::Endpoint,
+                             public ReceiveCoalescingExtension {
  public:
   SecureEndpoint(
       std::unique_ptr<grpc_event_engine::experimental::EventEngine::Endpoint>
@@ -855,9 +869,20 @@ class SecureEndpoint final : public EventEngine::Endpoint {
   }
 
   void* QueryExtension(absl::string_view id) override {
+    if (id == ReceiveCoalescingExtension::EndpointExtensionName()) {
+      return static_cast<ReceiveCoalescingExtension*>(this);
+    }
     return impl_->QueryExtension(id);
   }
 
+  void EnableRpcReceiveCoalescing() override {
+    impl_->EnableRpcReceiveCoalescing();
+  }
+
+  void DisableRpcReceiveCoalescing() override {
+    impl_->DisableRpcReceiveCoalescing();
+  }
+
   std::shared_ptr<TelemetryInfo> GetTelemetryInfo() const override {
     return std::make_shared<Impl::TelemetryInfo>(impl_->GetTelemetryInfo());
   }
@@ -940,11 +965,20 @@ class SecureEndpoint final : public EventEngine::Endpoint {
       // TODO(aananthv): Evaluate if we need to add a channel_arg to enable this
       // selectively.
       if (grpc_core::IsSecureEndpointReadCoalescingEnabled()) {
+        {
+          // This mutex should not have any contention since we can only have
+          // one outstanding Read() at a time.
+          grpc_core::MutexLock lock(&read_settings_mu_);
+          if (rpc_receive_coalescing_enabled_) {
+            // TODO(aananthv): Make required_read_bytes_ a separate field in
+            // ReadArgs to avoid confusion between min_progress_size and
+            // read_hint_bytes, especially if we enable coalescing by default.
+            required_read_bytes_ = std::max<int64_t>(0, args.read_hint_bytes());
+          } else {
+            required_read_bytes_ = 0;
+          }
+        }
         read_buffer_ = buffer;
-        // TODO(aananthv): Make required_read_bytes_ a separate field in
-        // ReadArgs to avoid confusion between min_progress_size and
-        // read_hint_bytes, especially if we enable coalescing by default.
-        required_read_bytes_ = std::max<int64_t>(0, args.read_hint_bytes());
         read_args_ = args;
         frame_protector_.BeginRead(buffer->c_slice_buffer(),
                                    required_read_bytes_);
@@ -1052,6 +1086,24 @@ class SecureEndpoint final : public EventEngine::Endpoint {
       return wrapped_ep_->GetLocalAddress();
     }
 
+    void EnableRpcReceiveCoalescing() {
+      {
+        grpc_core::MutexLock lock(&read_settings_mu_);
+        rpc_receive_coalescing_enabled_ = true;
+      }
+      frame_protector_.ResetReadStagingBuffer();
+      // We do not need to enable this in the underlying endpoint since we only
+      // benefit from coalescing the user-facing buffer.
+      // TODO(aananthv): Should we disable it explicitly?
+    }
+
+    void DisableRpcReceiveCoalescing() {
+      {
+        grpc_core::MutexLock lock(&read_settings_mu_);
+        rpc_receive_coalescing_enabled_ = false;
+      }
+    }
+
     void* QueryExtension(absl::string_view id) {
       return wrapped_ep_->QueryExtension(id);
     }
@@ -1306,6 +1358,9 @@ class SecureEndpoint final : public EventEngine::Endpoint {
     const size_t large_read_threshold_;
     const size_t large_write_threshold_;
     const size_t max_buffered_writes_;
+    grpc_core::Mutex read_settings_mu_;
+    bool rpc_receive_coalescing_enabled_ ABSL_GUARDED_BY(read_settings_mu_) =
+        false;
   };
 
   grpc_core::RefCountedPtr<Impl> impl_;

data/src/core/lib/channel/channel_args.h

@@ -55,6 +55,8 @@
 
 namespace grpc_core {
 
+class Arena;
+
 // Define a traits object for vtable lookup - allows us to integrate with
 // existing code easily (just define the trait!) and allows some magic in
 // ChannelArgs to automatically derive a vtable from a T*.
@@ -88,8 +90,7 @@ struct IsRawPointerTagged {
   static constexpr bool kValue = false;
 };
 template <typename T>
-struct IsRawPointerTagged<T,
-                          absl::void_t<typename T::RawPointerChannelArgTag>> {
+struct IsRawPointerTagged<T, std::void_t<typename T::RawPointerChannelArgTag>> {
   static constexpr bool kValue = true;
 };
 
@@ -114,8 +115,8 @@ template <typename T, typename = void>
 struct has_channel_args_compare : std::false_type {};
 template <typename T>
 struct has_channel_args_compare<
-    T, absl::void_t<decltype(T::ChannelArgsCompare(std::declval<const T*>(),
-                                                   std::declval<const T*>()))>>
+    T, std::void_t<decltype(T::ChannelArgsCompare(std::declval<const T*>(),
+                                                  std::declval<const T*>()))>>
     : std::true_type {};
 }  // namespace channel_args_detail
 
@@ -193,7 +194,7 @@ struct ChannelArgTypeTraits<
 // ownership*.
 template <typename T>
 struct ChannelArgTypeTraits<T,
-                            absl::void_t<typename T::RawPointerChannelArgTag>> {
+                            std::void_t<typename T::RawPointerChannelArgTag>> {
   static void* TakeUnownedPointer(T* p) { return p; }
   static const grpc_arg_pointer_vtable* VTable() {
     static const grpc_arg_pointer_vtable tbl = {
@@ -216,7 +217,7 @@ struct ChannelArgPointerShouldBeConst {
 
 template <typename T>
 struct ChannelArgPointerShouldBeConst<
-    T, absl::void_t<decltype(T::ChannelArgUseConstPtr())>> {
+    T, std::void_t<decltype(T::ChannelArgUseConstPtr())>> {
   static constexpr bool kValue = T::ChannelArgUseConstPtr();
 };
 
@@ -299,6 +300,11 @@ template <typename T>
 struct ChannelArgNameTraits<std::shared_ptr<T>> {
   static absl::string_view ChannelArgName() { return T::ChannelArgName(); }
 };
+template <>
+struct ChannelArgTypeTraits<Arena> {
+  static const grpc_arg_pointer_vtable* VTable();
+  static void* TakeUnownedPointer(Arena* p) { return p; }
+};
 // Specialization for the EventEngine
 template <>
 struct ChannelArgNameTraits<grpc_event_engine::experimental::EventEngine> {

data/src/core/lib/channel/channel_stack.cc

@@ -115,7 +115,7 @@ grpc_error_handle grpc_channel_stack_init(
     int initial_refs, grpc_iomgr_cb_func destroy, void* destroy_arg,
     std::vector<grpc_core::FilterAndConfig> filters,
     const grpc_core::ChannelArgs& channel_args, const char* name,
-    grpc_channel_stack* stack, const grpc_core::Blackboard* blackboard) {
+    grpc_channel_stack* stack) {
   if (GRPC_TRACE_FLAG_ENABLED(channel_stack)) {
     LOG(INFO) << "CHANNEL_STACK: init " << name;
     for (const auto& [filter, _] : filters) {
@@ -144,7 +144,6 @@ grpc_error_handle grpc_channel_stack_init(
                                              sizeof(grpc_channel_element));
 
   // init per-filter data
-  args.blackboard = blackboard;
   grpc_error_handle first_error;
   for (i = 0; i < filters.size(); ++i) {
     args.channel_stack = stack;

data/src/core/lib/channel/channel_stack.h

@@ -68,17 +68,12 @@
 #include "src/core/util/time_precise.h"
 #include "src/core/util/unique_type_name.h"
 
-namespace grpc_core {
-class Blackboard;
-}  // namespace grpc_core
-
 struct grpc_channel_element_args {
   grpc_channel_stack* channel_stack;
   grpc_core::ChannelArgs channel_args;
   int is_first;
   int is_last;
   grpc_core::RefCountedPtr<const grpc_core::FilterConfig> config;
-  const grpc_core::Blackboard* blackboard;
 };
 struct grpc_call_element_args {
   grpc_call_stack* call_stack;
@@ -276,8 +271,7 @@ grpc_error_handle grpc_channel_stack_init(
     int initial_refs, grpc_iomgr_cb_func destroy, void* destroy_arg,
     std::vector<grpc_core::FilterAndConfig> filters,
     const grpc_core::ChannelArgs& args, const char* name,
-    grpc_channel_stack* stack,
-    const grpc_core::Blackboard* blackboard = nullptr);
+    grpc_channel_stack* stack);
 // Destroy a channel stack
 void grpc_channel_stack_destroy(grpc_channel_stack* stack);
 

data/src/core/lib/channel/channel_stack_builder_impl.cc

@@ -70,7 +70,7 @@ ChannelStackBuilderImpl::Build() {
         grpc_channel_stack_destroy(stk);
         gpr_free(stk);
       },
-      channel_stack, stack, channel_args(), name(), channel_stack, blackboard_);
+      channel_stack, stack, channel_args(), name(), channel_stack);
 
   if (!error.ok()) {
     grpc_channel_stack_destroy(channel_stack);

data/src/core/lib/channel/channel_stack_builder_impl.h

@@ -24,8 +24,6 @@
 
 namespace grpc_core {
 
-class Blackboard;
-
 // Build a channel stack.
 // Allows interested parties to add filters to the stack, and to query an
 // in-progress build.
@@ -35,17 +33,12 @@ class ChannelStackBuilderImpl final : public ChannelStackBuilder {
  public:
   using ChannelStackBuilder::ChannelStackBuilder;
 
-  void SetBlackboard(const Blackboard* blackboard) { blackboard_ = blackboard; }
-
   // Build the channel stack.
   // After success, *result holds the new channel stack,
   // prefix_bytes are allocated before the channel stack,
   // initial_refs, destroy, destroy_arg are as per grpc_channel_stack_init
   // On failure, *result is nullptr.
   absl::StatusOr<RefCountedPtr<grpc_channel_stack>> Build() override;
-
- private:
-  const Blackboard* blackboard_ = nullptr;
 };
 
 }  // namespace grpc_core

data/src/core/lib/channel/promise_based_filter.cc

@@ -26,6 +26,7 @@
 #include "src/core/call/metadata.h"
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/debug/trace.h"
+#include "src/core/lib/experiments/experiments.h"
 #include "src/core/lib/iomgr/error.h"
 #include "src/core/lib/promise/seq.h"
 #include "src/core/lib/slice/slice.h"
@@ -33,6 +34,8 @@
 #include "src/core/util/grpc_check.h"
 #include "src/core/util/latent_see.h"
 #include "src/core/util/manual_constructor.h"
+#include "src/core/util/orphanable.h"
+#include "src/core/util/ref_counted.h"
 #include "src/core/util/status_helper.h"
 #include "absl/base/attributes.h"
 #include "absl/functional/function_ref.h"
@@ -81,6 +84,61 @@ absl::Status StatusFromMetadata(const ServerMetadata& md) {
 ///////////////////////////////////////////////////////////////////////////////
 // BaseCallData
 
+class BaseCallData::WeakWakerHandle final : public Wakeable, public Orphanable {
+ public:
+  explicit WeakWakerHandle(BaseCallData* base) : base_(base) {}
+
+  void Ref() { refs_.Ref(); }
+
+  void Orphan() override {
+    mu_.Lock();
+    base_ = nullptr;
+    mu_.Unlock();
+    Unref();
+  }
+
+  void Wakeup(WakeupMask wakeup_mask) override { WakeupGeneric(wakeup_mask); }
+
+  void WakeupAsync(WakeupMask wakeup_mask) override {
+    WakeupGeneric(wakeup_mask);
+  }
+
+  void Drop(WakeupMask) override { Unref(); }
+
+  std::string ActivityDebugTag(WakeupMask) const override {
+    MutexLock lock(&mu_);
+    return base_ == nullptr ? "<unknown>" : base_->DebugTag();
+  }
+
+ private:
+  void WakeupGeneric(WakeupMask wakeup_mask) {
+    BaseCallData* wakeup_base = nullptr;
+    {
+      MutexLock lock(&mu_);
+      if (base_ != nullptr) {
+        auto* call_stack = base_->call_stack();
+        if (call_stack->refcount.refs.RefIfNonZero(DEBUG_LOCATION, "waker")) {
+          wakeup_base = base_;
+        }
+      }
+    }
+    if (wakeup_base != nullptr) {
+      wakeup_base->Wakeup(wakeup_mask);
+    }
+    Unref();
+  }
+
+  void Unref() {
+    if (refs_.Unref()) {
+      delete this;
+    }
+  }
+
+  RefCount refs_;
+  mutable Mutex mu_;
+  BaseCallData* base_ ABSL_GUARDED_BY(mu_);
+};
+
 BaseCallData::BaseCallData(
     grpc_call_element* elem, const grpc_call_element_args* args, uint8_t flags,
     absl::FunctionRef<Interceptor*()> make_send_interceptor,
@@ -127,9 +185,18 @@ BaseCallData::~BaseCallData() {
 // Orphan().
 void BaseCallData::Orphan() { abort(); }
 
-// For now we don't care about owning/non-owning wakers, instead just share
-// implementation.
-Waker BaseCallData::MakeNonOwningWaker() { return MakeOwningWaker(); }
+Waker BaseCallData::MakeNonOwningWaker() {
+  if (IsV2NonOwningWakerImplementationEnabled()) {
+    if (handle_ == nullptr) {
+      handle_ = MakeOrphanable<WeakWakerHandle>(this);
+    }
+    handle_->Ref();
+    // The wakeup mask is unused by BaseCallData and WeakWakerHandle, so 0 is
+    // passed as a placeholder.
+    return Waker(handle_.get(), 0);
+  }
+  return MakeOwningWaker();
+}
 
 Waker BaseCallData::MakeOwningWaker() {
   GRPC_CALL_STACK_REF(call_stack_, "waker");
@@ -146,6 +213,14 @@ void BaseCallData::Wakeup(WakeupMask) {
   GRPC_CALL_COMBINER_START(call_combiner_, closure, absl::OkStatus(), "wakeup");
 }
 
+void BaseCallData::WakeupAsync(WakeupMask wakeup_mask) {
+  if (IsV2NonOwningWakerImplementationEnabled()) {
+    Wakeup(wakeup_mask);
+  } else {
+    Crash("not implemented");
+  }
+}
+
 void BaseCallData::Drop(WakeupMask) {
   GRPC_CALL_STACK_UNREF(call_stack_, "waker");
 }
@@ -495,6 +570,14 @@ void BaseCallData::SendMessage::Done(const ServerMetadata& metadata,
     case State::kPushedToPipe:
       push_.reset();
       next_.reset();
+      // Ensure captured transport batches are released if the call terminates
+      // while a message is in-flight. This prevents deadlocks where the
+      // transport never receives the batch, leaving the RPC hanging for
+      // metadata.
+      if (IsV2NonOwningWakerImplementationEnabled()) {
+        GRPC_DCHECK(batch_.is_captured());
+        batch_.CancelWith(StatusFromMetadata(metadata), flusher);
+      }
       state_ = State::kCancelledButNotYetPolled;
       if (base_->is_current()) base_->ForceImmediateRepoll();
       break;
@@ -1164,7 +1247,10 @@ class ClientCallData::PollContext {
                       std::exchange(
                           self_->recv_initial_metadata_->original_on_ready,
                           nullptr),
-                      absl::CancelledError(),
+                      IsV2NonOwningWakerImplementationEnabled() &&
+                              !StatusFromMetadata(*md).ok()
+                          ? StatusFromMetadata(*md)
+                          : absl::CancelledError(),
                       "wake_inside_combiner:recv_initial_metadata_ready");
               }
             }