grpc 1.60.2 → 1.61.0.pre2

data/src/core/lib/transport/transport.h

@@ -55,9 +55,13 @@
 #include "src/core/lib/promise/context.h"
 #include "src/core/lib/promise/detail/status.h"
 #include "src/core/lib/promise/latch.h"
+#include "src/core/lib/promise/party.h"
 #include "src/core/lib/promise/pipe.h"
+#include "src/core/lib/promise/race.h"
+#include "src/core/lib/promise/status_flag.h"
 #include "src/core/lib/resource_quota/arena.h"
 #include "src/core/lib/slice/slice_buffer.h"
+#include "src/core/lib/transport/call_final_info.h"
 #include "src/core/lib/transport/connectivity_state.h"
 #include "src/core/lib/transport/metadata_batch.h"
 #include "src/core/lib/transport/transport_fwd.h"
@@ -147,6 +151,17 @@ struct StatusCastImpl<ServerMetadataHandle, absl::Status&> {
   }
 };
 
+// Anything that can be first cast to absl::Status can then be cast to
+// ServerMetadataHandle.
+template <typename T>
+struct StatusCastImpl<
+    ServerMetadataHandle, T,
+    absl::void_t<decltype(StatusCast<absl::Status>(std::declval<T>()))>> {
+  static ServerMetadataHandle Cast(const T& m) {
+    return ServerMetadataFromStatus(StatusCast<absl::Status>(m));
+  }
+};
+
 // Move only type that tracks call startup.
 // Allows observation of when client_initial_metadata has been processed by the
 // end of the local call stack.
@@ -228,6 +243,367 @@ struct CallArgs {
 using NextPromiseFactory =
     std::function<ArenaPromise<ServerMetadataHandle>(CallArgs)>;
 
+// The common middle part of a call - a reference is held by each of
+// CallInitiator and CallHandler - which provide interfaces that are appropriate
+// for each side of a call.
+// The spine will ultimately host the pipes, filters, and context for one part
+// of a call: ie top-half client channel, sub channel call, server call.
+// TODO(ctiller): eventually drop this when we don't need to reference into
+// legacy promise calls anymore
+class CallSpineInterface {
+ public:
+  virtual ~CallSpineInterface() = default;
+  virtual Pipe<ClientMetadataHandle>& client_initial_metadata() = 0;
+  virtual Pipe<ServerMetadataHandle>& server_initial_metadata() = 0;
+  virtual Pipe<MessageHandle>& client_to_server_messages() = 0;
+  virtual Pipe<MessageHandle>& server_to_client_messages() = 0;
+  virtual Pipe<ServerMetadataHandle>& server_trailing_metadata() = 0;
+  virtual Latch<ServerMetadataHandle>& cancel_latch() = 0;
+  // Add a callback to be called when server trailing metadata is received.
+  void OnDone(absl::AnyInvocable<void()> fn) {
+    if (on_done_ == nullptr) {
+      on_done_ = std::move(fn);
+      return;
+    }
+    on_done_ = [first = std::move(fn), next = std::move(on_done_)]() mutable {
+      first();
+      next();
+    };
+  }
+  void CallOnDone() {
+    if (on_done_ != nullptr) std::exchange(on_done_, nullptr)();
+  }
+  virtual Party& party() = 0;
+  virtual void IncrementRefCount() = 0;
+  virtual void Unref() = 0;
+
+  // Cancel the call with the given metadata.
+  // Regarding the `MUST_USE_RESULT absl::nullopt_t`:
+  // Most cancellation calls right now happen in pipe interceptors;
+  // there `nullopt` indicates terminate processing of this pipe and close with
+  // error.
+  // It's convenient then to have the Cancel operation (setting the latch to
+  // terminate the call) be the last thing that occurs in a pipe interceptor,
+  // and this construction supports that (and has helped the author not write
+  // some bugs).
+  GRPC_MUST_USE_RESULT absl::nullopt_t Cancel(ServerMetadataHandle metadata) {
+    GPR_DEBUG_ASSERT(Activity::current() == &party());
+    auto& c = cancel_latch();
+    if (c.is_set()) return absl::nullopt;
+    c.Set(std::move(metadata));
+    CallOnDone();
+    client_initial_metadata().sender.CloseWithError();
+    server_initial_metadata().sender.CloseWithError();
+    client_to_server_messages().sender.CloseWithError();
+    server_to_client_messages().sender.CloseWithError();
+    return absl::nullopt;
+  }
+
+  auto WaitForCancel() {
+    GPR_DEBUG_ASSERT(Activity::current() == &party());
+    return cancel_latch().Wait();
+  }
+
+  // Wrap a promise so that if it returns failure it automatically cancels
+  // the rest of the call.
+  // The resulting (returned) promise will resolve to Empty.
+  template <typename Promise>
+  auto CancelIfFails(Promise promise) {
+    GPR_DEBUG_ASSERT(Activity::current() == &party());
+    using P = promise_detail::PromiseLike<Promise>;
+    using ResultType = typename P::Result;
+    return Map(std::move(promise), [this](ResultType r) {
+      if (!IsStatusOk(r)) {
+        std::ignore = Cancel(StatusCast<ServerMetadataHandle>(r));
+      }
+      return r;
+    });
+  }
+
+  // Spawn a promise that returns Empty{} and save some boilerplate handling
+  // that detail.
+  template <typename PromiseFactory>
+  void SpawnInfallible(absl::string_view name, PromiseFactory promise_factory) {
+    party().Spawn(name, std::move(promise_factory), [](Empty) {});
+  }
+
+  // Spawn a promise that returns some status-like type; if the status
+  // represents failure automatically cancel the rest of the call.
+  template <typename PromiseFactory>
+  void SpawnGuarded(absl::string_view name, PromiseFactory promise_factory) {
+    using FactoryType =
+        promise_detail::OncePromiseFactory<void, PromiseFactory>;
+    using PromiseType = typename FactoryType::Promise;
+    using ResultType = typename PromiseType::Result;
+    static_assert(
+        std::is_same<bool,
+                     decltype(IsStatusOk(std::declval<ResultType>()))>::value,
+        "SpawnGuarded promise must return a status-like object");
+    party().Spawn(name, std::move(promise_factory), [this](ResultType r) {
+      if (!IsStatusOk(r)) {
+        std::ignore = Cancel(StatusCast<ServerMetadataHandle>(std::move(r)));
+      }
+    });
+  }
+
+ private:
+  absl::AnyInvocable<void()> on_done_{nullptr};
+};
+
+class CallSpine final : public CallSpineInterface, public Party {
+ public:
+  static RefCountedPtr<CallSpine> Create(
+      grpc_event_engine::experimental::EventEngine* event_engine,
+      Arena* arena) {
+    return RefCountedPtr<CallSpine>(arena->New<CallSpine>(event_engine, arena));
+  }
+
+  Pipe<ClientMetadataHandle>& client_initial_metadata() override {
+    return client_initial_metadata_;
+  }
+  Pipe<ServerMetadataHandle>& server_initial_metadata() override {
+    return server_initial_metadata_;
+  }
+  Pipe<MessageHandle>& client_to_server_messages() override {
+    return client_to_server_messages_;
+  }
+  Pipe<MessageHandle>& server_to_client_messages() override {
+    return server_to_client_messages_;
+  }
+  Pipe<ServerMetadataHandle>& server_trailing_metadata() override {
+    return server_trailing_metadata_;
+  }
+  Latch<ServerMetadataHandle>& cancel_latch() override { return cancel_latch_; }
+  Party& party() override { return *this; }
+  void IncrementRefCount() override { Party::IncrementRefCount(); }
+  void Unref() override { Party::Unref(); }
+
+ private:
+  friend class Arena;
+  CallSpine(grpc_event_engine::experimental::EventEngine* event_engine,
+            Arena* arena)
+      : Party(arena, 1), event_engine_(event_engine) {}
+
+  class ScopedContext : public ScopedActivity,
+                        public promise_detail::Context<Arena> {
+   public:
+    explicit ScopedContext(CallSpine* spine)
+        : ScopedActivity(&spine->party()), Context<Arena>(spine->arena()) {}
+  };
+
+  bool RunParty() override {
+    ScopedContext context(this);
+    return Party::RunParty();
+  }
+
+  void PartyOver() override {
+    Arena* a = arena();
+    {
+      ScopedContext context(this);
+      CancelRemainingParticipants();
+      a->DestroyManagedNewObjects();
+    }
+    this->~CallSpine();
+    a->Destroy();
+  }
+
+  grpc_event_engine::experimental::EventEngine* event_engine() const override {
+    return event_engine_;
+  }
+
+  // Initial metadata from client to server
+  Pipe<ClientMetadataHandle> client_initial_metadata_{arena()};
+  // Initial metadata from server to client
+  Pipe<ServerMetadataHandle> server_initial_metadata_{arena()};
+  // Messages travelling from the application to the transport.
+  Pipe<MessageHandle> client_to_server_messages_{arena()};
+  // Messages travelling from the transport to the application.
+  Pipe<MessageHandle> server_to_client_messages_{arena()};
+  // Trailing metadata from server to client
+  Pipe<ServerMetadataHandle> server_trailing_metadata_{arena()};
+  // Latch that can be set to terminate the call
+  Latch<ServerMetadataHandle> cancel_latch_;
+  // Event engine associated with this call
+  grpc_event_engine::experimental::EventEngine* const event_engine_;
+};
+
+class CallInitiator {
+ public:
+  explicit CallInitiator(RefCountedPtr<CallSpineInterface> spine)
+      : spine_(std::move(spine)) {}
+
+  auto PushClientInitialMetadata(ClientMetadataHandle md) {
+    GPR_DEBUG_ASSERT(Activity::current() == &spine_->party());
+    return Map(spine_->client_initial_metadata().sender.Push(std::move(md)),
+               [](bool ok) { return StatusFlag(ok); });
+  }
+
+  auto PullServerInitialMetadata() {
+    GPR_DEBUG_ASSERT(Activity::current() == &spine_->party());
+    return Map(spine_->server_initial_metadata().receiver.Next(),
+               [](NextResult<ClientMetadataHandle> md)
+                   -> ValueOrFailure<ClientMetadataHandle> {
+                 if (!md.has_value()) return Failure{};
+                 return std::move(*md);
+               });
+  }
+
+  auto PullServerTrailingMetadata() {
+    GPR_DEBUG_ASSERT(Activity::current() == &spine_->party());
+    return Race(spine_->WaitForCancel(),
+                Map(spine_->server_trailing_metadata().receiver.Next(),
+                    [spine = spine_](NextResult<ServerMetadataHandle> md)
+                        -> ServerMetadataHandle {
+                      GPR_ASSERT(md.has_value());
+                      return std::move(*md);
+                    }));
+  }
+
+  auto PullMessage() {
+    GPR_DEBUG_ASSERT(Activity::current() == &spine_->party());
+    return spine_->server_to_client_messages().receiver.Next();
+  }
+
+  auto PushMessage(MessageHandle message) {
+    GPR_DEBUG_ASSERT(Activity::current() == &spine_->party());
+    return Map(
+        spine_->client_to_server_messages().sender.Push(std::move(message)),
+        [](bool r) { return StatusFlag(r); });
+  }
+
+  void FinishSends() {
+    GPR_DEBUG_ASSERT(Activity::current() == &spine_->party());
+    spine_->client_to_server_messages().sender.Close();
+  }
+
+  template <typename Promise>
+  auto CancelIfFails(Promise promise) {
+    return spine_->CancelIfFails(std::move(promise));
+  }
+
+  void Cancel() {
+    GPR_DEBUG_ASSERT(Activity::current() == &spine_->party());
+    std::ignore =
+        spine_->Cancel(ServerMetadataFromStatus(absl::CancelledError()));
+  }
+
+  template <typename PromiseFactory>
+  void SpawnGuarded(absl::string_view name, PromiseFactory promise_factory) {
+    spine_->SpawnGuarded(name, std::move(promise_factory));
+  }
+
+  template <typename PromiseFactory>
+  void SpawnInfallible(absl::string_view name, PromiseFactory promise_factory) {
+    spine_->SpawnInfallible(name, std::move(promise_factory));
+  }
+
+  template <typename PromiseFactory>
+  auto SpawnWaitable(absl::string_view name, PromiseFactory promise_factory) {
+    return spine_->party().SpawnWaitable(name, std::move(promise_factory));
+  }
+
+  Arena* arena() { return spine_->party().arena(); }
+
+ private:
+  RefCountedPtr<CallSpineInterface> spine_;
+};
+
+class CallHandler {
+ public:
+  explicit CallHandler(RefCountedPtr<CallSpineInterface> spine)
+      : spine_(std::move(spine)) {}
+
+  auto PullClientInitialMetadata() {
+    GPR_DEBUG_ASSERT(Activity::current() == &spine_->party());
+    return Map(spine_->client_initial_metadata().receiver.Next(),
+               [](NextResult<ClientMetadataHandle> md)
+                   -> ValueOrFailure<ClientMetadataHandle> {
+                 if (!md.has_value()) return Failure{};
+                 return std::move(*md);
+               });
+  }
+
+  auto PushServerInitialMetadata(ServerMetadataHandle md) {
+    GPR_DEBUG_ASSERT(Activity::current() == &spine_->party());
+    return Map(spine_->server_initial_metadata().sender.Push(std::move(md)),
+               [](bool ok) { return StatusFlag(ok); });
+  }
+
+  auto PushServerTrailingMetadata(ServerMetadataHandle md) {
+    GPR_DEBUG_ASSERT(Activity::current() == &spine_->party());
+    spine_->server_to_client_messages().sender.Close();
+    spine_->CallOnDone();
+    return Map(spine_->server_trailing_metadata().sender.Push(std::move(md)),
+               [](bool ok) { return StatusFlag(ok); });
+  }
+
+  auto PullMessage() {
+    GPR_DEBUG_ASSERT(Activity::current() == &spine_->party());
+    return spine_->client_to_server_messages().receiver.Next();
+  }
+
+  auto PushMessage(MessageHandle message) {
+    GPR_DEBUG_ASSERT(Activity::current() == &spine_->party());
+    return Map(
+        spine_->server_to_client_messages().sender.Push(std::move(message)),
+        [](bool ok) { return StatusFlag(ok); });
+  }
+
+  void Cancel(ServerMetadataHandle status) {
+    GPR_DEBUG_ASSERT(Activity::current() == &spine_->party());
+    std::ignore = spine_->Cancel(std::move(status));
+  }
+
+  void OnDone(absl::AnyInvocable<void()> fn) { spine_->OnDone(std::move(fn)); }
+
+  template <typename Promise>
+  auto CancelIfFails(Promise promise) {
+    return spine_->CancelIfFails(std::move(promise));
+  }
+
+  template <typename PromiseFactory>
+  void SpawnGuarded(absl::string_view name, PromiseFactory promise_factory) {
+    spine_->SpawnGuarded(name, std::move(promise_factory));
+  }
+
+  template <typename PromiseFactory>
+  void SpawnInfallible(absl::string_view name, PromiseFactory promise_factory) {
+    spine_->SpawnInfallible(name, std::move(promise_factory));
+  }
+
+  template <typename PromiseFactory>
+  auto SpawnWaitable(absl::string_view name, PromiseFactory promise_factory) {
+    return spine_->party().SpawnWaitable(name, std::move(promise_factory));
+  }
+
+  Arena* arena() { return spine_->party().arena(); }
+
+ private:
+  RefCountedPtr<CallSpineInterface> spine_;
+};
+
+struct CallInitiatorAndHandler {
+  CallInitiator initiator;
+  CallHandler handler;
+};
+
+CallInitiatorAndHandler MakeCall(
+    grpc_event_engine::experimental::EventEngine* event_engine, Arena* arena);
+
+template <typename CallHalf>
+auto OutgoingMessages(CallHalf h) {
+  struct Wrapper {
+    CallHalf h;
+    auto Next() { return h.PullMessage(); }
+  };
+  return Wrapper{std::move(h)};
+}
+
+// Forward a call from `call_handler` to `call_initiator` (with initial metadata
+// `client_initial_metadata`)
+void ForwardCall(CallHandler call_handler, CallInitiator call_initiator,
+                 ClientMetadataHandle client_initial_metadata);
+
 }  // namespace grpc_core
 
 // forward declarations
@@ -304,24 +680,6 @@ inline void grpc_stream_unref(grpc_stream_refcount* refcount) {
 grpc_slice grpc_slice_from_stream_owned_buffer(grpc_stream_refcount* refcount,
                                                void* buffer, size_t length);
 
-struct grpc_transport_one_way_stats {
-  uint64_t framing_bytes = 0;
-  uint64_t data_bytes = 0;
-  uint64_t header_bytes = 0;
-};
-
-struct grpc_transport_stream_stats {
-  grpc_transport_one_way_stats incoming;
-  grpc_transport_one_way_stats outgoing;
-  gpr_timespec latency = gpr_inf_future(GPR_TIMESPAN);
-};
-
-void grpc_transport_move_one_way_stats(grpc_transport_one_way_stats* from,
-                                       grpc_transport_one_way_stats* to);
-
-void grpc_transport_move_stats(grpc_transport_stream_stats* from,
-                               grpc_transport_stream_stats* to);
-
 // This struct (which is present in both grpc_transport_stream_op_batch
 // and grpc_transport_op_batch) is a convenience to allow filters or
 // transports to schedule a closure related to a particular batch without
@@ -633,9 +991,7 @@ class FilterStackTransport {
 
 class ClientTransport {
  public:
-  // Create a promise to execute one client call.
-  virtual ArenaPromise<ServerMetadataHandle> MakeCallPromise(
-      CallArgs call_args) = 0;
+  virtual void StartCall(CallHandler call_handler) = 0;
 
  protected:
   ~ClientTransport() = default;
@@ -643,10 +999,24 @@ class ClientTransport {
 
 class ServerTransport {
  public:
-  // Register the factory function for the filter stack part of a call
-  // promise.
-  void SetCallPromiseFactory(
-      absl::AnyInvocable<ArenaPromise<ServerMetadataHandle>(CallArgs) const>);
+  // Acceptor helps transports create calls.
+  class Acceptor {
+   public:
+    // Returns an arena that can be used to allocate memory for initial metadata
+    // parsing, and later passed to CreateCall() as the underlying arena for
+    // that call.
+    virtual Arena* CreateArena() = 0;
+    // Create a call at the server (or fail)
+    // arena must have been previously allocated by CreateArena()
+    virtual absl::StatusOr<CallInitiator> CreateCall(
+        ClientMetadata& client_initial_metadata, Arena* arena) = 0;
+
+   protected:
+    ~Acceptor() = default;
+  };
+
+  // Called once slightly after transport setup to register the accept function.
+  virtual void SetAcceptor(Acceptor* acceptor) = 0;
 
  protected:
   ~ServerTransport() = default;

data/src/core/plugin_registry/grpc_plugin_registry.cc

@@ -40,6 +40,8 @@ extern void SecurityRegisterHandshakerFactories(
     CoreConfiguration::Builder* builder);
 extern void RegisterClientAuthorityFilter(CoreConfiguration::Builder* builder);
 extern void RegisterChannelIdleFilters(CoreConfiguration::Builder* builder);
+extern void RegisterLegacyChannelIdleFilters(
+    CoreConfiguration::Builder* builder);
 extern void RegisterDeadlineFilter(CoreConfiguration::Builder* builder);
 extern void RegisterGrpcLbPolicy(CoreConfiguration::Builder* builder);
 extern void RegisterHttpFilters(CoreConfiguration::Builder* builder);
@@ -89,6 +91,7 @@ void BuildCoreConfiguration(CoreConfiguration::Builder* builder) {
   SecurityRegisterHandshakerFactories(builder);
   RegisterClientAuthorityFilter(builder);
   RegisterChannelIdleFilters(builder);
+  RegisterLegacyChannelIdleFilters(builder);
   RegisterConnectedChannel(builder);
   RegisterGrpcLbPolicy(builder);
   RegisterHttpFilters(builder);

data/src/core/plugin_registry/grpc_plugin_registry_extra.cc

@@ -34,8 +34,6 @@ extern void RegisterXdsClusterManagerLbPolicy(
     CoreConfiguration::Builder* builder);
 extern void RegisterXdsClusterImplLbPolicy(CoreConfiguration::Builder* builder);
 extern void RegisterCdsLbPolicy(CoreConfiguration::Builder* builder);
-extern void RegisterXdsClusterResolverLbPolicy(
-    CoreConfiguration::Builder* builder);
 extern void RegisterXdsOverrideHostLbPolicy(
     CoreConfiguration::Builder* builder);
 extern void RegisterXdsWrrLocalityLbPolicy(CoreConfiguration::Builder* builder);
@@ -58,7 +56,6 @@ void RegisterExtraFilters(CoreConfiguration::Builder* builder) {
   RegisterXdsClusterManagerLbPolicy(builder);
   RegisterXdsClusterImplLbPolicy(builder);
   RegisterCdsLbPolicy(builder);
-  RegisterXdsClusterResolverLbPolicy(builder);
   RegisterXdsOverrideHostLbPolicy(builder);
   RegisterXdsWrrLocalityLbPolicy(builder);
   RegisterRingHashLbPolicy(builder);

data/src/core/tsi/alts/handshaker/alts_handshaker_client.cc

@@ -23,7 +23,7 @@
 #include <list>
 
 #include "absl/strings/numbers.h"
-#include "upb/upb.hpp"
+#include "upb/mem/arena.hpp"
 
 #include <grpc/byte_buffer.h>
 #include <grpc/support/alloc.h>

data/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc

@@ -24,7 +24,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "upb/upb.hpp"
+#include "upb/mem/arena.hpp"
 
 #include <grpc/grpc_security.h>
 #include <grpc/support/alloc.h>

data/src/core/tsi/alts/handshaker/transport_security_common_api.cc

@@ -20,7 +20,7 @@
 
 #include "src/core/tsi/alts/handshaker/transport_security_common_api.h"
 
-#include "upb/upb.hpp"
+#include "upb/mem/arena.hpp"
 
 bool grpc_gcp_rpc_protocol_versions_set_max(
     grpc_gcp_rpc_protocol_versions* versions, uint32_t max_major,