grpc 1.52.0 → 1.53.0

data/src/core/lib/channel/promise_based_filter.h

@@ -12,22 +12,20 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef GRPC_CORE_LIB_CHANNEL_PROMISE_BASED_FILTER_H
-#define GRPC_CORE_LIB_CHANNEL_PROMISE_BASED_FILTER_H
+#ifndef GRPC_SRC_CORE_LIB_CHANNEL_PROMISE_BASED_FILTER_H
+#define GRPC_SRC_CORE_LIB_CHANNEL_PROMISE_BASED_FILTER_H
 
 // Scaffolding to allow the per-call part of a filter to be authored in a
 // promise-style. Most of this will be removed once the promises conversion is
 // completed.
 
-// TODO(ctiller): When removing this file, also reduce the number of *'s on the
-// server initial metadata latch.
-
 #include <grpc/support/port_platform.h>
 
 #include <stdint.h>
 #include <stdlib.h>
 
 #include <atomic>
+#include <initializer_list>
 #include <memory>
 #include <new>
 #include <string>
@@ -35,6 +33,7 @@
 #include <utility>
 
 #include "absl/container/inlined_vector.h"
+#include "absl/functional/function_ref.h"
 #include "absl/meta/type_traits.h"
 #include "absl/status/status.h"
 #include "absl/strings/string_view.h"
@@ -45,7 +44,6 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/channel/call_finalization.h"
-#include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_fwd.h"
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/channel/context.h"
@@ -60,7 +58,6 @@
 #include "src/core/lib/promise/activity.h"
 #include "src/core/lib/promise/arena_promise.h"
 #include "src/core/lib/promise/context.h"
-#include "src/core/lib/promise/latch.h"
 #include "src/core/lib/promise/pipe.h"
 #include "src/core/lib/promise/poll.h"
 #include "src/core/lib/resource_quota/arena.h"
@@ -119,7 +116,7 @@ class ChannelFilter {
   }
 
  private:
-  // TODO(ctiller): remove once per-channel-stack event engines land
+  // TODO(ctiller): remove once per-channel-stack EventEngines land
   std::shared_ptr<grpc_event_engine::experimental::EventEngine> event_engine_ =
       grpc_event_engine::experimental::GetDefaultEventEngine();
 };
@@ -152,9 +149,27 @@ class InvalidChannelFilter : public ChannelFilter {
 
 // Call data shared between all implementations of promise-based filters.
 class BaseCallData : public Activity, private Wakeable {
- public:
+ protected:
+  // Hook to allow interception of messages on the send/receive path by
+  // PipeSender and PipeReceiver, as appropriate according to whether we're
+  // client or server.
+  class Interceptor {
+   public:
+    virtual PipeSender<MessageHandle>* Push() = 0;
+    virtual PipeReceiver<MessageHandle>* Pull() = 0;
+    virtual PipeReceiver<MessageHandle>* original_receiver() = 0;
+    virtual PipeSender<MessageHandle>* original_sender() = 0;
+    virtual void GotPipe(PipeSender<MessageHandle>*) = 0;
+    virtual void GotPipe(PipeReceiver<MessageHandle>*) = 0;
+    virtual ~Interceptor() = default;
+  };
+
   BaseCallData(grpc_call_element* elem, const grpc_call_element_args* args,
-               uint8_t flags);
+               uint8_t flags,
+               absl::FunctionRef<Interceptor*()> make_send_interceptor,
+               absl::FunctionRef<Interceptor*()> make_recv_interceptor);
+
+ public:
   ~BaseCallData() override;
 
   void set_pollent(grpc_polling_entity* pollent) {
@@ -167,7 +182,7 @@ class BaseCallData : public Activity, private Wakeable {
   Waker MakeNonOwningWaker() final;
   Waker MakeOwningWaker() final;
 
-  std::string ActivityDebugTag() const override { return DebugTag(); }
+  std::string ActivityDebugTag(void*) const override { return DebugTag(); }
 
   void Finalize(const grpc_call_final_info* final_info) {
     finalization_.Run(final_info);
@@ -270,6 +285,60 @@ class BaseCallData : public Activity, private Wakeable {
     return p.release();
   }
 
+  class ReceiveInterceptor final : public Interceptor {
+   public:
+    explicit ReceiveInterceptor(Arena* arena) : pipe_{arena} {}
+
+    PipeReceiver<MessageHandle>* original_receiver() override {
+      return &pipe_.receiver;
+    }
+    PipeSender<MessageHandle>* original_sender() override { abort(); }
+
+    void GotPipe(PipeReceiver<MessageHandle>* receiver) override {
+      GPR_ASSERT(receiver_ == nullptr);
+      receiver_ = receiver;
+    }
+
+    void GotPipe(PipeSender<MessageHandle>*) override { abort(); }
+
+    PipeSender<MessageHandle>* Push() override { return &pipe_.sender; }
+    PipeReceiver<MessageHandle>* Pull() override {
+      GPR_ASSERT(receiver_ != nullptr);
+      return receiver_;
+    }
+
+   private:
+    Pipe<MessageHandle> pipe_;
+    PipeReceiver<MessageHandle>* receiver_ = nullptr;
+  };
+
+  class SendInterceptor final : public Interceptor {
+   public:
+    explicit SendInterceptor(Arena* arena) : pipe_{arena} {}
+
+    PipeReceiver<MessageHandle>* original_receiver() override { abort(); }
+    PipeSender<MessageHandle>* original_sender() override {
+      return &pipe_.sender;
+    }
+
+    void GotPipe(PipeReceiver<MessageHandle>*) override { abort(); }
+
+    void GotPipe(PipeSender<MessageHandle>* sender) override {
+      GPR_ASSERT(sender_ == nullptr);
+      sender_ = sender;
+    }
+
+    PipeSender<MessageHandle>* Push() override {
+      GPR_ASSERT(sender_ != nullptr);
+      return sender_;
+    }
+    PipeReceiver<MessageHandle>* Pull() override { return &pipe_.receiver; }
+
+   private:
+    Pipe<MessageHandle> pipe_;
+    PipeSender<MessageHandle>* sender_ = nullptr;
+  };
+
   // State machine for sending messages: handles intercepting send_message ops
   // and forwarding them through pipes to the promise, then getting the result
   // down the stack.
@@ -277,9 +346,11 @@ class BaseCallData : public Activity, private Wakeable {
   // these members for filters that don't need to intercept sent messages.
   class SendMessage {
    public:
-    explicit SendMessage(BaseCallData* base)
-        : base_(base), pipe_(base->arena()) {}
-    PipeReceiver<MessageHandle>* outgoing_pipe() { return &pipe_.receiver; }
+    SendMessage(BaseCallData* base, Interceptor* interceptor)
+        : base_(base), interceptor_(interceptor) {}
+    ~SendMessage() { interceptor_->~Interceptor(); }
+
+    Interceptor* interceptor() { return interceptor_; }
 
     // Start a send_message op.
     void StartOp(CapturedBatch batch);
@@ -287,16 +358,19 @@ class BaseCallData : public Activity, private Wakeable {
     // This happens when the promise requests to call the next filter: until
     // this occurs messages can't be sent as we don't know the pipe that the
     // promise expects to send on.
-    void GotPipe(PipeReceiver<MessageHandle>* receiver);
+    template <typename T>
+    void GotPipe(T* pipe);
     // Called from client/server polling to do the send message part of the
     // work.
-    void WakeInsideCombiner(Flusher* flusher);
+    void WakeInsideCombiner(Flusher* flusher, bool allow_push_to_pipe);
     // Call is completed, we have trailing metadata. Close things out.
-    void Done(const ServerMetadata& metadata);
+    void Done(const ServerMetadata& metadata, Flusher* flusher);
     // Return true if we have a batch captured (for debug logs)
     bool HaveCapturedBatch() const { return batch_.is_captured(); }
     // Return true if we're not actively sending a message.
     bool IsIdle() const;
+    // Return true if we've released the message for forwarding down the stack.
+    bool IsForwarded() const { return state_ == State::kForwardedBatch; }
 
    private:
     enum class State : uint8_t {
@@ -331,11 +405,9 @@ class BaseCallData : public Activity, private Wakeable {
 
     BaseCallData* const base_;
     State state_ = State::kInitial;
-    Pipe<MessageHandle> pipe_;
-    PipeReceiver<MessageHandle>* receiver_ = nullptr;
+    Interceptor* const interceptor_;
     absl::optional<PipeSender<MessageHandle>::PushType> push_;
-    absl::optional<PipeReceiver<MessageHandle>::NextType> next_;
-    absl::optional<NextResult<MessageHandle>> next_result_;
+    absl::optional<PipeReceiverNextType<MessageHandle>> next_;
     CapturedBatch batch_;
     grpc_closure* intercepted_on_complete_;
     grpc_closure on_complete_ =
@@ -351,9 +423,11 @@ class BaseCallData : public Activity, private Wakeable {
   // these members for filters that don't need to intercept sent messages.
   class ReceiveMessage {
    public:
-    explicit ReceiveMessage(BaseCallData* base)
-        : base_(base), pipe_(base->arena()) {}
-    PipeSender<MessageHandle>* incoming_pipe() { return &pipe_.sender; }
+    ReceiveMessage(BaseCallData* base, Interceptor* interceptor)
+        : base_(base), interceptor_(interceptor) {}
+    ~ReceiveMessage() { interceptor_->~Interceptor(); }
+
+    Interceptor* interceptor() { return interceptor_; }
 
     // Start a recv_message op.
     void StartOp(CapturedBatch& batch);
@@ -361,10 +435,11 @@ class BaseCallData : public Activity, private Wakeable {
     // This happens when the promise requests to call the next filter: until
     // this occurs messages can't be received as we don't know the pipe that the
     // promise expects to forward them with.
-    void GotPipe(PipeSender<MessageHandle>* sender);
+    template <typename T>
+    void GotPipe(T* pipe);
     // Called from client/server polling to do the receive message part of the
     // work.
-    void WakeInsideCombiner(Flusher* flusher);
+    void WakeInsideCombiner(Flusher* flusher, bool allow_push_to_pipe);
     // Call is completed, we have trailing metadata. Close things out.
     void Done(const ServerMetadata& metadata, Flusher* flusher);
 
@@ -403,28 +478,37 @@ class BaseCallData : public Activity, private Wakeable {
       // stack: we need to keep track of that until we get the completion so
       // that we do the right thing in OnComplete.
       kCancelledWhilstForwarding,
+      // The same, but before we got the pipe
+      kCancelledWhilstForwardingNoPipe,
       // Call got terminated whilst we had a recv_message batch completed, and
       // we've now received the completion.
       // On the next poll we'll close things out and forward on completions,
       // then transition to cancelled.
       kBatchCompletedButCancelled,
-      // Completed successfully while we're processing a recv message.
+      // The same, but before we got the pipe
+      kBatchCompletedButCancelledNoPipe,
+      // Completed successfully while we're processing a recv message - see
+      // kPushedToPipe.
       kCompletedWhilePushedToPipe,
+      // Completed successfully while we're processing a recv message - see
+      // kPulledFromPipe.
       kCompletedWhilePulledFromPipe,
+      // Completed successfully while we were waiting to process
+      // kBatchCompleted.
+      kCompletedWhileBatchCompleted,
     };
     static const char* StateString(State);
 
     void OnComplete(absl::Status status);
 
     BaseCallData* const base_;
-    Pipe<MessageHandle> pipe_;
-    PipeSender<MessageHandle>* sender_;
+    Interceptor* const interceptor_;
     State state_ = State::kInitial;
     uint32_t scratch_flags_;
     absl::optional<SliceBuffer>* intercepted_slice_buffer_;
     uint32_t* intercepted_flags_;
     absl::optional<PipeSender<MessageHandle>::PushType> push_;
-    absl::optional<PipeReceiver<MessageHandle>::NextType> next_;
+    absl::optional<PipeReceiverNextType<MessageHandle>> next_;
     absl::Status completed_status_;
     grpc_closure* intercepted_on_complete_;
     grpc_closure on_complete_ =
@@ -436,15 +520,8 @@ class BaseCallData : public Activity, private Wakeable {
   CallCombiner* call_combiner() const { return call_combiner_; }
   Timestamp deadline() const { return deadline_; }
   grpc_call_stack* call_stack() const { return call_stack_; }
-  Latch<ServerMetadata*>* server_initial_metadata_latch() const {
-    return server_initial_metadata_latch_;
-  }
-  PipeReceiver<MessageHandle>* outgoing_messages_pipe() const {
-    return send_message_ == nullptr ? nullptr : send_message_->outgoing_pipe();
-  }
-  PipeSender<MessageHandle>* incoming_messages_pipe() const {
-    return receive_message_ == nullptr ? nullptr
-                                       : receive_message_->incoming_pipe();
+  Pipe<ServerMetadataHandle>* server_initial_metadata_pipe() const {
+    return server_initial_metadata_pipe_;
   }
   SendMessage* send_message() const { return send_message_; }
   ReceiveMessage* receive_message() const { return receive_message_; }
@@ -461,8 +538,8 @@ class BaseCallData : public Activity, private Wakeable {
 
  private:
   // Wakeable implementation.
-  void Wakeup() final;
-  void Drop() final;
+  void Wakeup(void*) final;
+  void Drop(void*) final;
 
   virtual void OnWakeup() = 0;
 
@@ -474,7 +551,7 @@ class BaseCallData : public Activity, private Wakeable {
   CallFinalization finalization_;
   grpc_call_context_element* const context_;
   std::atomic<grpc_polling_entity*> pollent_{nullptr};
-  Latch<ServerMetadata*>* const server_initial_metadata_latch_;
+  Pipe<ServerMetadataHandle>* const server_initial_metadata_pipe_;
   SendMessage* const send_message_;
   ReceiveMessage* const receive_message_;
   grpc_event_engine::experimental::EventEngine* event_engine_;
@@ -626,6 +703,9 @@ class ServerCallData : public BaseCallData {
     // metadata.
     kQueuedBehindSendMessage,
     // We saw the op, and are waiting for the promise to complete
+    // to forward it. First however we need to close sends.
+    kQueuedButHaventClosedSends,
+    // We saw the op, and are waiting for the promise to complete
     // to forward it.
     kQueued,
     // We've forwarded the op to the next filter.
@@ -792,7 +872,7 @@ struct ChannelFilterWithFlagsMethods {
   static absl::Status InitChannelElem(grpc_channel_element* elem,
                                       grpc_channel_element_args* args) {
     GPR_ASSERT(args->is_last == ((kFlags & kFilterIsLast) != 0));
-    auto status = F::Create(ChannelArgs::FromC(args->channel_args),
+    auto status = F::Create(args->channel_args,
                             ChannelFilter::Args(args->channel_stack, elem));
     if (!status.ok()) {
       static_assert(
@@ -854,4 +934,4 @@ MakePromiseBasedFilter(const char* name) {
 
 }  // namespace grpc_core
 
-#endif  // GRPC_CORE_LIB_CHANNEL_PROMISE_BASED_FILTER_H
+#endif  // GRPC_SRC_CORE_LIB_CHANNEL_PROMISE_BASED_FILTER_H

data/src/core/lib/channel/status_util.h

@@ -16,8 +16,8 @@
 //
 //
 
-#ifndef GRPC_CORE_LIB_CHANNEL_STATUS_UTIL_H
-#define GRPC_CORE_LIB_CHANNEL_STATUS_UTIL_H
+#ifndef GRPC_SRC_CORE_LIB_CHANNEL_STATUS_UTIL_H
+#define GRPC_SRC_CORE_LIB_CHANNEL_STATUS_UTIL_H
 
 #include <grpc/support/port_platform.h>
 
@@ -78,4 +78,4 @@ absl::Status MaybeRewriteIllegalStatusCode(absl::Status status,
 
 }  // namespace grpc_core
 
-#endif  // GRPC_CORE_LIB_CHANNEL_STATUS_UTIL_H
+#endif  // GRPC_SRC_CORE_LIB_CHANNEL_STATUS_UTIL_H

data/src/core/lib/compression/compression_internal.h

@@ -16,8 +16,8 @@
 //
 //
 
-#ifndef GRPC_CORE_LIB_COMPRESSION_COMPRESSION_INTERNAL_H
-#define GRPC_CORE_LIB_COMPRESSION_COMPRESSION_INTERNAL_H
+#ifndef GRPC_SRC_CORE_LIB_COMPRESSION_COMPRESSION_INTERNAL_H
+#define GRPC_SRC_CORE_LIB_COMPRESSION_COMPRESSION_INTERNAL_H
 
 #include <grpc/support/port_platform.h>
 
@@ -90,4 +90,4 @@ class CompressionAlgorithmSet {
 
 }  // namespace grpc_core
 
-#endif  // GRPC_CORE_LIB_COMPRESSION_COMPRESSION_INTERNAL_H
+#endif  // GRPC_SRC_CORE_LIB_COMPRESSION_COMPRESSION_INTERNAL_H

data/src/core/lib/compression/message_compress.h

@@ -16,8 +16,8 @@
 //
 //
 
-#ifndef GRPC_CORE_LIB_COMPRESSION_MESSAGE_COMPRESS_H
-#define GRPC_CORE_LIB_COMPRESSION_MESSAGE_COMPRESS_H
+#ifndef GRPC_SRC_CORE_LIB_COMPRESSION_MESSAGE_COMPRESS_H
+#define GRPC_SRC_CORE_LIB_COMPRESSION_MESSAGE_COMPRESS_H
 
 #include <grpc/support/port_platform.h>
 
@@ -36,4 +36,4 @@ int grpc_msg_compress(grpc_compression_algorithm algorithm,
 int grpc_msg_decompress(grpc_compression_algorithm algorithm,
                         grpc_slice_buffer* input, grpc_slice_buffer* output);
 
-#endif  // GRPC_CORE_LIB_COMPRESSION_MESSAGE_COMPRESS_H
+#endif  // GRPC_SRC_CORE_LIB_COMPRESSION_MESSAGE_COMPRESS_H

data/src/core/lib/config/core_configuration.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef GRPC_CORE_LIB_CONFIG_CORE_CONFIGURATION_H
-#define GRPC_CORE_LIB_CONFIG_CORE_CONFIGURATION_H
+#ifndef GRPC_SRC_CORE_LIB_CONFIG_CORE_CONFIGURATION_H
+#define GRPC_SRC_CORE_LIB_CONFIG_CORE_CONFIGURATION_H
 
 #include <grpc/support/port_platform.h>
 
@@ -240,4 +240,4 @@ extern void BuildCoreConfiguration(CoreConfiguration::Builder* builder);
 
 }  // namespace grpc_core
 
-#endif  // GRPC_CORE_LIB_CONFIG_CORE_CONFIGURATION_H
+#endif  // GRPC_SRC_CORE_LIB_CONFIG_CORE_CONFIGURATION_H

data/src/core/lib/debug/event_log.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef GRPC_CORE_LIB_DEBUG_EVENT_LOG_H
-#define GRPC_CORE_LIB_DEBUG_EVENT_LOG_H
+#ifndef GRPC_SRC_CORE_LIB_DEBUG_EVENT_LOG_H
+#define GRPC_SRC_CORE_LIB_DEBUG_EVENT_LOG_H
 
 #include <grpc/support/port_platform.h>
 
@@ -78,4 +78,4 @@ class EventLog {
 
 }  // namespace grpc_core
 
-#endif  // GRPC_CORE_LIB_DEBUG_EVENT_LOG_H
+#endif  // GRPC_SRC_CORE_LIB_DEBUG_EVENT_LOG_H

data/src/core/lib/debug/histogram_view.h

@@ -12,8 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef GRPC_CORE_LIB_DEBUG_HISTOGRAM_VIEW_H
-#define GRPC_CORE_LIB_DEBUG_HISTOGRAM_VIEW_H
+#ifndef GRPC_SRC_CORE_LIB_DEBUG_HISTOGRAM_VIEW_H
+#define GRPC_SRC_CORE_LIB_DEBUG_HISTOGRAM_VIEW_H
 
 #include <grpc/support/port_platform.h>
 
@@ -34,4 +34,4 @@ struct HistogramView {
 
 }  // namespace grpc_core
 
-#endif  // GRPC_CORE_LIB_DEBUG_HISTOGRAM_VIEW_H
+#endif  // GRPC_SRC_CORE_LIB_DEBUG_HISTOGRAM_VIEW_H

data/src/core/lib/debug/stats.h

@@ -16,8 +16,8 @@
 //
 //
 
-#ifndef GRPC_CORE_LIB_DEBUG_STATS_H
-#define GRPC_CORE_LIB_DEBUG_STATS_H
+#ifndef GRPC_SRC_CORE_LIB_DEBUG_STATS_H
+#define GRPC_SRC_CORE_LIB_DEBUG_STATS_H
 
 #include <grpc/support/port_platform.h>
 
@@ -61,4 +61,4 @@ std::string StatsAsJson(T* data) {
 
 }  // namespace grpc_core
 
-#endif  // GRPC_CORE_LIB_DEBUG_STATS_H
+#endif  // GRPC_SRC_CORE_LIB_DEBUG_STATS_H