grpc 1.60.0 → 1.62.0

data/src/core/lib/channel/channel_args.h

@@ -125,6 +125,16 @@ struct ChannelArgTypeTraits<
   };
 };
 
+// Define a check for shared_ptr supported types, which must extend
+// enable_shared_from_this.
+template <typename T>
+struct SupportedSharedPtrType
+    : std::integral_constant<
+          bool, std::is_base_of<std::enable_shared_from_this<T>, T>::value> {};
+template <>
+struct SupportedSharedPtrType<grpc_event_engine::experimental::EventEngine>
+    : std::true_type {};
+
 // Specialization for shared_ptr
 // Incurs an allocation because shared_ptr.release is not a thing.
 template <typename T>
@@ -173,19 +183,27 @@ struct ChannelArgTypeTraits<T,
   };
 };
 
-// GetObject support for shared_ptr and RefCountedPtr
+// Determine if the pointer for a channel arg name should be const or not
+template <typename T, typename SfinaeVoid = void>
+struct ChannelArgPointerShouldBeConst {
+  static constexpr bool kValue = false;
+};
+
 template <typename T>
-struct WrapInSharedPtr
-    : std::integral_constant<
-          bool, std::is_base_of<std::enable_shared_from_this<T>, T>::value> {};
-template <>
-struct WrapInSharedPtr<grpc_event_engine::experimental::EventEngine>
-    : std::true_type {};
+struct ChannelArgPointerShouldBeConst<
+    T, absl::void_t<decltype(T::ChannelArgUseConstPtr())>> {
+  static constexpr bool kValue = T::ChannelArgUseConstPtr();
+};
+
+// GetObject support for shared_ptr and RefCountedPtr
 template <typename T, typename Ignored = void /* for SFINAE */>
 struct GetObjectImpl;
 // std::shared_ptr implementation
 template <typename T>
-struct GetObjectImpl<T, absl::enable_if_t<WrapInSharedPtr<T>::value, void>> {
+struct GetObjectImpl<
+    T, absl::enable_if_t<!ChannelArgPointerShouldBeConst<T>::kValue &&
+                             SupportedSharedPtrType<T>::value,
+                         void>> {
   using Result = T*;
   using ReffedResult = std::shared_ptr<T>;
   using StoredType = std::shared_ptr<T>*;
@@ -205,11 +223,34 @@ struct GetObjectImpl<T, absl::enable_if_t<WrapInSharedPtr<T>::value, void>> {
 };
 // RefCountedPtr
 template <typename T>
-struct GetObjectImpl<T, absl::enable_if_t<!WrapInSharedPtr<T>::value, void>> {
+struct GetObjectImpl<
+    T, absl::enable_if_t<!ChannelArgPointerShouldBeConst<T>::kValue &&
+                             !SupportedSharedPtrType<T>::value,
+                         void>> {
   using Result = T*;
   using ReffedResult = RefCountedPtr<T>;
   using StoredType = Result;
   static Result Get(StoredType p) { return p; };
+  static ReffedResult GetReffed(StoredType p) {
+    if (p == nullptr) return nullptr;
+    return p->template RefAsSubclass<T>();
+  };
+  static ReffedResult GetReffed(StoredType p, const DebugLocation& location,
+                                const char* reason) {
+    if (p == nullptr) return nullptr;
+    return p->template RefAsSubclass<T>(location, reason);
+  };
+};
+
+template <typename T>
+struct GetObjectImpl<
+    T, absl::enable_if_t<ChannelArgPointerShouldBeConst<T>::kValue &&
+                             !SupportedSharedPtrType<T>::value,
+                         void>> {
+  using Result = const T*;
+  using ReffedResult = RefCountedPtr<const T>;
+  using StoredType = Result;
+  static Result Get(StoredType p) { return p; };
   static ReffedResult GetReffed(StoredType p) {
     if (p == nullptr) return nullptr;
     return p->Ref();
@@ -237,6 +278,7 @@ struct ChannelArgNameTraits<grpc_event_engine::experimental::EventEngine> {
     return GRPC_INTERNAL_ARG_EVENT_ENGINE;
   }
 };
+
 class ChannelArgs {
  public:
   class Pointer {
@@ -282,9 +324,23 @@ class ChannelArgs {
     const grpc_arg_pointer_vtable* vtable_;
   };
 
+  // Helper to create a `Pointer` object to an object that is not owned by the
+  // `ChannelArgs` object. Useful for tests, a code smell for production code.
+  template <typename T>
+  static Pointer UnownedPointer(T* p) {
+    static const grpc_arg_pointer_vtable vtable = {
+        [](void* p) -> void* { return p; },
+        [](void*) {},
+        [](void* p, void* q) { return QsortCompare(p, q); },
+    };
+    return Pointer(p, &vtable);
+  }
+
   class Value {
    public:
-    explicit Value(int n) : rep_(reinterpret_cast<void*>(n), &int_vtable_) {}
+    explicit Value(int n)
+        : rep_(reinterpret_cast<void*>(static_cast<intptr_t>(n)),
+               &int_vtable_) {}
     explicit Value(std::string s)
         : rep_(RefCountedString::Make(s).release(), &string_vtable_) {}
     explicit Value(Pointer p) : rep_(std::move(p)) {}
@@ -303,7 +359,7 @@ class ChannelArgs {
       return &rep_;
     }
 
-    std::string ToString() const;
+    absl::string_view ToString(std::list<std::string>& backing) const;
 
     grpc_arg MakeCArg(const char* name) const;
 
@@ -376,21 +432,38 @@ class ChannelArgs {
   GRPC_MUST_USE_RESULT auto Set(absl::string_view name,
                                 RefCountedPtr<T> value) const
       -> absl::enable_if_t<
-          std::is_same<const grpc_arg_pointer_vtable*,
-                       decltype(ChannelArgTypeTraits<
-                                absl::remove_cvref_t<T>>::VTable())>::value,
+          !ChannelArgPointerShouldBeConst<T>::kValue &&
+              std::is_same<const grpc_arg_pointer_vtable*,
+                           decltype(ChannelArgTypeTraits<
+                                    absl::remove_cvref_t<T>>::VTable())>::value,
           ChannelArgs> {
     return Set(
         name, Pointer(value.release(),
                       ChannelArgTypeTraits<absl::remove_cvref_t<T>>::VTable()));
   }
   template <typename T>
+  GRPC_MUST_USE_RESULT auto Set(absl::string_view name,
+                                RefCountedPtr<const T> value) const
+      -> absl::enable_if_t<
+          ChannelArgPointerShouldBeConst<T>::kValue &&
+              std::is_same<const grpc_arg_pointer_vtable*,
+                           decltype(ChannelArgTypeTraits<
+                                    absl::remove_cvref_t<T>>::VTable())>::value,
+          ChannelArgs> {
+    return Set(
+        name, Pointer(const_cast<T*>(value.release()),
+                      ChannelArgTypeTraits<absl::remove_cvref_t<T>>::VTable()));
+  }
+  template <typename T>
   GRPC_MUST_USE_RESULT absl::enable_if_t<
       std::is_same<
           const grpc_arg_pointer_vtable*,
           decltype(ChannelArgTypeTraits<std::shared_ptr<T>>::VTable())>::value,
       ChannelArgs>
   Set(absl::string_view name, std::shared_ptr<T> value) const {
+    static_assert(SupportedSharedPtrType<T>::value,
+                  "Type T must extend std::enable_shared_from_this to be added "
+                  "into ChannelArgs as a shared_ptr<T>");
     auto* store_value = new std::shared_ptr<T>(value);
     return Set(
         name,
@@ -418,6 +491,8 @@ class ChannelArgs {
   absl::optional<int> GetInt(absl::string_view name) const;
   absl::optional<absl::string_view> GetString(absl::string_view name) const;
   absl::optional<std::string> GetOwnedString(absl::string_view name) const;
+  // WARNING: this is broken if `name` represents something that was stored as a
+  // RefCounted<const T> - we will discard the const-ness.
   void* GetVoidPointer(absl::string_view name) const;
   template <typename T>
   typename GetObjectImpl<T>::StoredType GetPointer(

data/src/core/lib/channel/channel_stack.cc

@@ -28,6 +28,7 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/channel_fwd.h"
 #include "src/core/lib/channel/channel_stack_trace.h"
 #include "src/core/lib/gpr/alloc.h"
 #include "src/core/lib/surface/channel_init.h"
@@ -320,3 +321,29 @@ grpc_channel_stack::MakeServerCallPromise(grpc_core::CallArgs call_args) {
   return ServerNext(grpc_channel_stack_element(this, this->count - 1))(
       std::move(call_args));
 }
+
+void grpc_channel_stack::InitClientCallSpine(
+    grpc_core::CallSpineInterface* call) {
+  for (size_t i = 0; i < count; i++) {
+    auto* elem = grpc_channel_stack_element(this, i);
+    if (elem->filter->init_call == nullptr) {
+      grpc_core::Crash(
+          absl::StrCat("Filter '", elem->filter->name,
+                       "' does not support the call-v3 interface"));
+    }
+    elem->filter->init_call(elem, call);
+  }
+}
+
+void grpc_channel_stack::InitServerCallSpine(
+    grpc_core::CallSpineInterface* call) {
+  for (size_t i = 0; i < count; i++) {
+    auto* elem = grpc_channel_stack_element(this, count - 1 - i);
+    if (elem->filter->init_call == nullptr) {
+      grpc_core::Crash(
+          absl::StrCat("Filter '", elem->filter->name,
+                       "' does not support the call-v3 interface"));
+    }
+    elem->filter->init_call(elem, call);
+  }
+}

data/src/core/lib/channel/channel_stack.h

@@ -72,6 +72,7 @@
 #include "src/core/lib/iomgr/polling_entity.h"
 #include "src/core/lib/promise/arena_promise.h"
 #include "src/core/lib/resource_quota/arena.h"
+#include "src/core/lib/transport/call_final_info.h"
 #include "src/core/lib/transport/transport.h"
 
 struct grpc_channel_element_args {
@@ -90,16 +91,6 @@ struct grpc_call_element_args {
   grpc_core::Arena* arena;
   grpc_core::CallCombiner* call_combiner;
 };
-struct grpc_call_stats {
-  grpc_transport_stream_stats transport_stream_stats;
-  gpr_timespec latency;  // From call creating to enqueing of received status
-};
-/// Information about the call upon completion.
-struct grpc_call_final_info {
-  grpc_call_stats stats;
-  grpc_status_code final_status = GRPC_STATUS_OK;
-  const char* error_string = nullptr;
-};
 
 // Channel filters specify:
 // 1. the amount of memory needed in the channel & call (via the sizeof_XXX
@@ -128,6 +119,12 @@ struct grpc_channel_filter {
   grpc_core::ArenaPromise<grpc_core::ServerMetadataHandle> (*make_call_promise)(
       grpc_channel_element* elem, grpc_core::CallArgs call_args,
       grpc_core::NextPromiseFactory next_promise_factory);
+  // Register interceptors into a call.
+  // If this is non-null it may be used in preference to make_call_promise.
+  // There is an on-going migration to move all filters to providing this, and
+  // then to drop start_transport_stream_op_batch.
+  void (*init_call)(grpc_channel_element* elem,
+                    grpc_core::CallSpineInterface* call_spine);
   // Called to handle channel level operations - e.g. new calls, or transport
   // closure.
   // See grpc_channel_next_op on how to call the next element in the stack
@@ -238,6 +235,9 @@ struct grpc_channel_stack {
   MakeClientCallPromise(grpc_core::CallArgs call_args);
   grpc_core::ArenaPromise<grpc_core::ServerMetadataHandle>
   MakeServerCallPromise(grpc_core::CallArgs call_args);
+
+  void InitClientCallSpine(grpc_core::CallSpineInterface* call);
+  void InitServerCallSpine(grpc_core::CallSpineInterface* call);
 };
 
 // A call stack tracks a set of related filters for one call, and guarantees

data/src/core/lib/channel/connected_channel.cc

@@ -364,7 +364,7 @@ class ConnectedChannelStream : public Orphanable {
   grpc_stream_refcount stream_refcount_;
   StreamPtr stream_;
   Arena* arena_ = GetContext<Arena>();
-  Party* const party_ = static_cast<Party*>(Activity::current());
+  Party* const party_ = GetContext<Party>();
   ExternallyObservableLatch<void> finished_;
 };
 
@@ -383,17 +383,18 @@ auto ConnectedChannelStream::RecvMessages(
               gpr_log(GPR_INFO,
                       "%s[connected] RecvMessage: received payload of %" PRIdPTR
                       " bytes",
-                      Activity::current()->DebugTag().c_str(),
+                      GetContext<Activity>()->DebugTag().c_str(),
                       pending_message->payload()->Length());
             }
             return Map(incoming_messages.Push(std::move(pending_message)),
                        [](bool ok) -> LoopCtl<absl::Status> {
                          if (!ok) {
                            if (grpc_call_trace.enabled()) {
-                             gpr_log(GPR_INFO,
-                                     "%s[connected] RecvMessage: failed to "
-                                     "push message towards the application",
-                                     Activity::current()->DebugTag().c_str());
+                             gpr_log(
+                                 GPR_INFO,
+                                 "%s[connected] RecvMessage: failed to "
+                                 "push message towards the application",
+                                 GetContext<Activity>()->DebugTag().c_str());
                            }
                            return absl::OkStatus();
                          }
@@ -406,11 +407,13 @@ auto ConnectedChannelStream::RecvMessages(
               gpr_log(GPR_INFO,
                       "%s[connected] RecvMessage: reached end of stream with "
                       "status:%s",
-                      Activity::current()->DebugTag().c_str(),
+                      GetContext<Activity>()->DebugTag().c_str(),
                       status.status().ToString().c_str());
             }
             if (cancel_on_error && !status.ok()) {
               incoming_messages.CloseWithError();
+            } else {
+              incoming_messages.Close();
             }
             return Immediate(LoopCtl<absl::Status>(status.status()));
           };
@@ -442,7 +445,7 @@ ArenaPromise<ServerMetadataHandle> MakeClientCallPromise(Transport* transport,
   transport->filter_stack_transport()->InitStream(stream->stream(),
                                                   stream->stream_refcount(),
                                                   nullptr, GetContext<Arena>());
-  auto* party = static_cast<Party*>(Activity::current());
+  auto* party = GetContext<Party>();
   party->Spawn("set_polling_entity", call_args.polling_entity->Wait(),
                [transport, stream = stream->InternalRef()](
                    grpc_polling_entity polling_entity) {
@@ -472,7 +475,7 @@ ArenaPromise<ServerMetadataHandle> MakeClientCallPromise(Transport* transport,
                if (grpc_call_trace.enabled()) {
                  gpr_log(GPR_DEBUG,
                          "%s[connected] Publish client initial metadata: %s",
-                         Activity::current()->DebugTag().c_str(),
+                         GetContext<Activity>()->DebugTag().c_str(),
                          server_initial_metadata->DebugString().c_str());
                }
                return Map(pipe->Push(std::move(server_initial_metadata)),
@@ -579,7 +582,7 @@ ArenaPromise<ServerMetadataHandle> MakeServerCallPromise(
       stream->stream(), stream->stream_refcount(),
       GetContext<CallContext>()->server_call_context()->server_stream_data(),
       GetContext<Arena>());
-  auto* party = static_cast<Party*>(Activity::current());
+  auto* party = GetContext<Party>();
 
   // Arifacts we need for the lifetime of the call.
   struct CallData {
@@ -745,7 +748,7 @@ ArenaPromise<ServerMetadataHandle> MakeServerCallPromise(
                       : nullptr;
         if (md != nullptr) {
           call_data->sent_initial_metadata = true;
-          auto* party = static_cast<Party*>(Activity::current());
+          auto* party = GetContext<Party>();
           party->Spawn("connected/send_initial_metadata",
                        GetContext<BatchBuilder>()->SendServerInitialMetadata(
                            stream->batch_target(), std::move(md)),
@@ -775,7 +778,7 @@ ArenaPromise<ServerMetadataHandle> MakeServerCallPromise(
               gpr_log(
                   GPR_DEBUG,
                   "%s[connected] Got trailing metadata; status=%s metadata=%s",
-                  Activity::current()->DebugTag().c_str(),
+                  GetContext<Activity>()->DebugTag().c_str(),
                   status.status().ToString().c_str(),
                   status.ok() ? (*status)->DebugString().c_str() : "<none>");
             }
@@ -857,6 +860,7 @@ grpc_channel_filter MakeConnectedFilter() {
   return {
       connected_channel_start_transport_stream_op_batch,
       make_call_promise != nullptr ? make_call_wrapper : nullptr,
+      /* init_call: */ nullptr,
       connected_channel_start_transport_op,
       sizeof(call_data),
       connected_channel_init_call_elem,
@@ -870,10 +874,12 @@ grpc_channel_filter MakeConnectedFilter() {
         // do this, and I'm not sure what that is yet. This is only "safe"
         // because call stacks place no additional data after the last call
         // element, and the last call element MUST be the connected channel.
-        channel_stack->call_stack_size +=
-            static_cast<channel_data*>(elem->channel_data)
-                ->transport->filter_stack_transport()
-                ->SizeOfStream();
+        auto* transport =
+            static_cast<channel_data*>(elem->channel_data)->transport;
+        if (transport->filter_stack_transport() != nullptr) {
+          channel_stack->call_stack_size +=
+              transport->filter_stack_transport()->SizeOfStream();
+        }
       },
       connected_channel_destroy_channel_elem,
       connected_channel_get_channel_info,
@@ -881,13 +887,27 @@ grpc_channel_filter MakeConnectedFilter() {
   };
 }
 
-ArenaPromise<ServerMetadataHandle> MakeTransportCallPromise(
+ArenaPromise<ServerMetadataHandle> MakeClientTransportCallPromise(
     Transport* transport, CallArgs call_args, NextPromiseFactory) {
-  return transport->client_transport()->MakeCallPromise(std::move(call_args));
+  auto spine = GetContext<CallContext>()->MakeCallSpine(std::move(call_args));
+  transport->client_transport()->StartCall(CallHandler{spine});
+  return Map(spine->server_trailing_metadata().receiver.Next(),
+             [](NextResult<ServerMetadataHandle> r) {
+               if (r.has_value()) {
+                 auto md = std::move(r.value());
+                 md->Set(GrpcStatusFromWire(), true);
+                 return md;
+               }
+               auto m = GetContext<Arena>()->MakePooled<ServerMetadata>(
+                   GetContext<Arena>());
+               m->Set(GrpcStatusMetadata(), GRPC_STATUS_CANCELLED);
+               m->Set(GrpcCallWasCancelled(), true);
+               return m;
+             });
 }
 
-const grpc_channel_filter kPromiseBasedTransportFilter =
-    MakeConnectedFilter<MakeTransportCallPromise>();
+const grpc_channel_filter kClientPromiseBasedTransportFilter =
+    MakeConnectedFilter<MakeClientTransportCallPromise>();
 
 #ifdef GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL
 const grpc_channel_filter kClientEmulatedFilter =
@@ -905,11 +925,37 @@ const grpc_channel_filter kServerEmulatedFilter =
     MakeConnectedFilter<nullptr>();
 #endif
 
-bool TransportSupportsPromiseBasedCalls(const ChannelArgs& args) {
+// noop filter for the v3 stack: placeholder for now because other code requires
+// we have a terminator.
+// TODO(ctiller): delete when v3 transition is complete.
+const grpc_channel_filter kServerPromiseBasedTransportFilter = {
+    nullptr,
+    [](grpc_channel_element*, CallArgs, NextPromiseFactory)
+        -> ArenaPromise<ServerMetadataHandle> { Crash("not implemented"); },
+    /* init_call: */ [](grpc_channel_element*, CallSpineInterface*) {},
+    connected_channel_start_transport_op,
+    0,
+    nullptr,
+    set_pollset_or_pollset_set,
+    nullptr,
+    sizeof(channel_data),
+    connected_channel_init_channel_elem,
+    +[](grpc_channel_stack*, grpc_channel_element*) {},
+    connected_channel_destroy_channel_elem,
+    connected_channel_get_channel_info,
+    "connected",
+};
+
+bool TransportSupportsClientPromiseBasedCalls(const ChannelArgs& args) {
   auto* transport = args.GetObject<Transport>();
   return transport->client_transport() != nullptr;
 }
 
+bool TransportSupportsServerPromiseBasedCalls(const ChannelArgs& args) {
+  auto* transport = args.GetObject<Transport>();
+  return transport->server_transport() != nullptr;
+}
+
 }  // namespace
 
 void RegisterConnectedChannel(CoreConfiguration::Builder* builder) {
@@ -922,32 +968,33 @@ void RegisterConnectedChannel(CoreConfiguration::Builder* builder) {
   // Option 1, and our ideal: the transport supports promise based calls,
   // and so we simply use the transport directly.
   builder->channel_init()
-      ->RegisterFilter(GRPC_CLIENT_SUBCHANNEL, &kPromiseBasedTransportFilter)
+      ->RegisterFilter(GRPC_CLIENT_SUBCHANNEL,
+                       &kClientPromiseBasedTransportFilter)
       .Terminal()
-      .If(TransportSupportsPromiseBasedCalls);
+      .If(TransportSupportsClientPromiseBasedCalls);
   builder->channel_init()
       ->RegisterFilter(GRPC_CLIENT_DIRECT_CHANNEL,
-                       &kPromiseBasedTransportFilter)
+                       &kClientPromiseBasedTransportFilter)
       .Terminal()
-      .If(TransportSupportsPromiseBasedCalls);
+      .If(TransportSupportsClientPromiseBasedCalls);
   builder->channel_init()
-      ->RegisterFilter(GRPC_SERVER_CHANNEL, &kPromiseBasedTransportFilter)
+      ->RegisterFilter(GRPC_SERVER_CHANNEL, &kServerPromiseBasedTransportFilter)
       .Terminal()
-      .If(TransportSupportsPromiseBasedCalls);
+      .If(TransportSupportsServerPromiseBasedCalls);
 
   // Option 2: the transport does not support promise based calls.
   builder->channel_init()
       ->RegisterFilter(GRPC_CLIENT_SUBCHANNEL, &kClientEmulatedFilter)
       .Terminal()
-      .IfNot(TransportSupportsPromiseBasedCalls);
+      .IfNot(TransportSupportsClientPromiseBasedCalls);
   builder->channel_init()
       ->RegisterFilter(GRPC_CLIENT_DIRECT_CHANNEL, &kClientEmulatedFilter)
       .Terminal()
-      .IfNot(TransportSupportsPromiseBasedCalls);
+      .IfNot(TransportSupportsClientPromiseBasedCalls);
   builder->channel_init()
       ->RegisterFilter(GRPC_SERVER_CHANNEL, &kServerEmulatedFilter)
       .Terminal()
-      .IfNot(TransportSupportsPromiseBasedCalls);
+      .IfNot(TransportSupportsServerPromiseBasedCalls);
 }
 
 }  // namespace grpc_core

data/src/core/lib/channel/promise_based_filter.cc

@@ -374,11 +374,11 @@ void BaseCallData::SendMessage::GotPipe(T* pipe_end) {
   switch (state_) {
     case State::kInitial:
       state_ = State::kIdle;
-      Activity::current()->ForceImmediateRepoll();
+      GetContext<Activity>()->ForceImmediateRepoll();
       break;
     case State::kGotBatchNoPipe:
       state_ = State::kGotBatch;
-      Activity::current()->ForceImmediateRepoll();
+      GetContext<Activity>()->ForceImmediateRepoll();
       break;
     case State::kIdle:
     case State::kGotBatch:
@@ -567,7 +567,7 @@ void BaseCallData::SendMessage::WakeInsideCombiner(Flusher* flusher,
       }
       if (completed_status_.ok()) {
         state_ = State::kIdle;
-        Activity::current()->ForceImmediateRepoll();
+        GetContext<Activity>()->ForceImmediateRepoll();
       } else {
         state_ = State::kCancelled;
       }
@@ -675,7 +675,7 @@ void BaseCallData::ReceiveMessage::GotPipe(T* pipe_end) {
       break;
     case State::kBatchCompletedNoPipe:
       state_ = State::kBatchCompleted;
-      Activity::current()->ForceImmediateRepoll();
+      GetContext<Activity>()->ForceImmediateRepoll();
       break;
     case State::kIdle:
     case State::kForwardedBatch: