grpc 1.45.0 → 1.46.3

data/src/core/lib/channel/promise_based_filter.h

@@ -27,6 +27,7 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/channel/call_finalization.h"
+#include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/channel_stack.h"
 #include "src/core/lib/channel/context.h"
 #include "src/core/lib/gprpp/debug_location.h"
@@ -42,21 +43,25 @@ class ChannelFilter {
  public:
   class Args {
    public:
-    Args() : Args(nullptr) {}
-    explicit Args(grpc_channel_stack* channel_stack)
-        : channel_stack_(channel_stack) {}
+    Args() : Args(nullptr, nullptr) {}
+    explicit Args(grpc_channel_stack* channel_stack,
+                  grpc_channel_element* channel_element)
+        : channel_stack_(channel_stack), channel_element_(channel_element) {}
 
     grpc_channel_stack* channel_stack() const { return channel_stack_; }
+    grpc_channel_element* uninitialized_channel_element() {
+      return channel_element_;
+    }
 
    private:
     friend class ChannelFilter;
     grpc_channel_stack* channel_stack_;
+    grpc_channel_element* channel_element_;
   };
 
   // Construct a promise for one call.
-  virtual ArenaPromise<TrailingMetadata> MakeCallPromise(
-      ClientInitialMetadata initial_metadata,
-      NextPromiseFactory next_promise_factory) = 0;
+  virtual ArenaPromise<ServerMetadataHandle> MakeCallPromise(
+      CallArgs call_args, NextPromiseFactory next_promise_factory) = 0;
 
   // Start a legacy transport op
   // Return true if the op was handled, false if it should be passed to the
@@ -65,7 +70,6 @@ class ChannelFilter {
   // structures going forward.
   virtual bool StartTransportOp(grpc_transport_op*) { return false; }
 
- protected:
   virtual ~ChannelFilter() = default;
 };
 
@@ -77,20 +81,22 @@ enum class FilterEndpoint {
   kServer,
 };
 
+// Flags for MakePromiseBasedFilter.
+static constexpr uint8_t kFilterExaminesServerInitialMetadata = 1;
+
 namespace promise_filter_detail {
 
 // Call data shared between all implementations of promise-based filters.
 class BaseCallData : public Activity, private Wakeable {
  public:
-  BaseCallData(grpc_call_element* elem, const grpc_call_element_args* args)
-      : call_stack_(args->call_stack),
-        elem_(elem),
-        arena_(args->arena),
-        call_combiner_(args->call_combiner),
-        deadline_(args->deadline),
-        context_(args->context) {}
+  BaseCallData(grpc_call_element* elem, const grpc_call_element_args* args,
+               uint8_t flags);
+  ~BaseCallData() override;
 
-  void set_pollent(grpc_polling_entity* pollent) { pollent_ = pollent; }
+  void set_pollent(grpc_polling_entity* pollent) {
+    GPR_ASSERT(nullptr ==
+               pollent_.exchange(pollent, std::memory_order_release));
+  }
 
   // Activity implementation (partial).
   void Orphan() final;
@@ -112,7 +118,8 @@ class BaseCallData : public Activity, private Wakeable {
         : promise_detail::Context<Arena>(call_data->arena_),
           promise_detail::Context<grpc_call_context_element>(
               call_data->context_),
-          promise_detail::Context<grpc_polling_entity>(call_data->pollent_),
+          promise_detail::Context<grpc_polling_entity>(
+              call_data->pollent_.load(std::memory_order_acquire)),
           promise_detail::Context<CallFinalization>(&call_data->finalization_) {
     }
   };
@@ -127,10 +134,14 @@ class BaseCallData : public Activity, private Wakeable {
     return p.Unwrap();
   }
 
+  Arena* arena() { return arena_; }
   grpc_call_element* elem() const { return elem_; }
   CallCombiner* call_combiner() const { return call_combiner_; }
   Timestamp deadline() const { return deadline_; }
   grpc_call_stack* call_stack() const { return call_stack_; }
+  Latch<ServerMetadata*>* server_initial_metadata_latch() const {
+    return server_initial_metadata_latch_;
+  }
 
  private:
   // Wakeable implementation.
@@ -146,12 +157,14 @@ class BaseCallData : public Activity, private Wakeable {
   const Timestamp deadline_;
   CallFinalization finalization_;
   grpc_call_context_element* const context_;
-  grpc_polling_entity* pollent_ = nullptr;
+  std::atomic<grpc_polling_entity*> pollent_{nullptr};
+  Latch<ServerMetadata*>* server_initial_metadata_latch_ = nullptr;
 };
 
 class ClientCallData : public BaseCallData {
  public:
-  ClientCallData(grpc_call_element* elem, const grpc_call_element_args* args);
+  ClientCallData(grpc_call_element* elem, const grpc_call_element_args* args,
+                 uint8_t flags);
   ~ClientCallData() override;
 
   // Activity implementation.
@@ -192,6 +205,9 @@ class ClientCallData : public BaseCallData {
     kCancelled
   };
 
+  struct RecvInitialMetadata;
+  class PollContext;
+
   // Handle cancellation.
   void Cancel(grpc_error_handle error);
   // Begin running the promise - which will ultimately take some initial
@@ -205,17 +221,17 @@ class ClientCallData : public BaseCallData {
   // Effectively:
   //   - put the modified initial metadata into the batch to be sent down.
   //   - return a wrapper around PollTrailingMetadata as the promise.
-  ArenaPromise<TrailingMetadata> MakeNextPromise(
-      ClientInitialMetadata initial_metadata);
+  ArenaPromise<ServerMetadataHandle> MakeNextPromise(CallArgs call_args);
   // Wrapper to make it look like we're calling the next filter as a promise.
   // First poll: send the send_initial_metadata op down the stack.
   // All polls: await receiving the trailing metadata, then return it to the
   // application.
-  Poll<TrailingMetadata> PollTrailingMetadata();
+  Poll<ServerMetadataHandle> PollTrailingMetadata();
   static void RecvTrailingMetadataReadyCallback(void* arg,
                                                 grpc_error_handle error);
   void RecvTrailingMetadataReady(grpc_error_handle error);
-  // Given an error, fill in TrailingMetadata to represent that error.
+  void RecvInitialMetadataReady(grpc_error_handle error);
+  // Given an error, fill in ServerMetadataHandle to represent that error.
   void SetStatusFromError(grpc_metadata_batch* metadata,
                           grpc_error_handle error);
   // Wakeup and poll the promise if appropriate.
@@ -223,11 +239,13 @@ class ClientCallData : public BaseCallData {
   void OnWakeup() override;
 
   // Contained promise
-  ArenaPromise<TrailingMetadata> promise_;
+  ArenaPromise<ServerMetadataHandle> promise_;
   // Queued batch containing at least a send_initial_metadata op.
   grpc_transport_stream_op_batch* send_initial_metadata_batch_ = nullptr;
   // Pointer to where trailing metadata will be stored.
   grpc_metadata_batch* recv_trailing_metadata_ = nullptr;
+  // State tracking recv initial metadata for filters that care about it.
+  RecvInitialMetadata* recv_initial_metadata_ = nullptr;
   // Closure to call when we're done with the trailing metadata.
   grpc_closure* original_recv_trailing_metadata_ready_ = nullptr;
   // Our closure pointing to RecvTrailingMetadataReadyCallback.
@@ -238,17 +256,14 @@ class ClientCallData : public BaseCallData {
   SendInitialState send_initial_state_ = SendInitialState::kInitial;
   // State of the recv_trailing_metadata op.
   RecvTrailingState recv_trailing_state_ = RecvTrailingState::kInitial;
-  // Whether we're currently polling the promise.
-  bool is_polling_ = false;
-  // Should we repoll after completing polling?
-  bool repoll_ = false;
-  // Whether we should forward send initial metadata after polling?
-  bool forward_send_initial_metadata_ = false;
+  // Polling related data. Non-null if we're actively polling
+  PollContext* poll_ctx_ = nullptr;
 };
 
 class ServerCallData : public BaseCallData {
  public:
-  ServerCallData(grpc_call_element* elem, const grpc_call_element_args* args);
+  ServerCallData(grpc_call_element* elem, const grpc_call_element_args* args,
+                 uint8_t flags);
   ~ServerCallData() override;
 
   // Activity implementation.
@@ -289,12 +304,11 @@ class ServerCallData : public BaseCallData {
   // Effectively:
   //   - put the modified initial metadata into the batch being sent up.
   //   - return a wrapper around PollTrailingMetadata as the promise.
-  ArenaPromise<TrailingMetadata> MakeNextPromise(
-      ClientInitialMetadata initial_metadata);
+  ArenaPromise<ServerMetadataHandle> MakeNextPromise(CallArgs call_args);
   // Wrapper to make it look like we're calling the next filter as a promise.
   // All polls: await sending the trailing metadata, then foward it down the
   // stack.
-  Poll<TrailingMetadata> PollTrailingMetadata();
+  Poll<ServerMetadataHandle> PollTrailingMetadata();
   static void RecvInitialMetadataReadyCallback(void* arg,
                                                grpc_error_handle error);
   void RecvInitialMetadataReady(grpc_error_handle error);
@@ -303,7 +317,7 @@ class ServerCallData : public BaseCallData {
   void OnWakeup() override;
 
   // Contained promise
-  ArenaPromise<TrailingMetadata> promise_;
+  ArenaPromise<ServerMetadataHandle> promise_;
   // Pointer to where initial metadata will be stored.
   grpc_metadata_batch* recv_initial_metadata_ = nullptr;
   // Closure to call when we're done with the trailing metadata.
@@ -351,11 +365,11 @@ class CallData<ChannelFilter, FilterEndpoint::kServer> : public ServerCallData {
 // class SomeChannelFilter : public ChannelFilter {
 //  public:
 //   static absl::StatusOr<SomeChannelFilter> Create(
-//       ChannelFilter::Args filter_args);
+//       ChannelArgs channel_args, ChannelFilter::Args filter_args);
 // };
 // TODO(ctiller): allow implementing get_channel_info, start_transport_op in
 // some way on ChannelFilter.
-template <typename F, FilterEndpoint kEndpoint>
+template <typename F, FilterEndpoint kEndpoint, uint8_t kFlags = 0>
 absl::enable_if_t<std::is_base_of<ChannelFilter, F>::value, grpc_channel_filter>
 MakePromiseBasedFilter(const char* name) {
   using CallData = promise_filter_detail::CallData<F, kEndpoint>;
@@ -366,10 +380,10 @@ MakePromiseBasedFilter(const char* name) {
         static_cast<CallData*>(elem->call_data)->StartBatch(batch);
       },
       // make_call_promise
-      [](grpc_channel_element* elem, ClientInitialMetadata initial_metadata,
+      [](grpc_channel_element* elem, CallArgs call_args,
          NextPromiseFactory next_promise_factory) {
         return static_cast<F*>(elem->channel_data)
-            ->MakeCallPromise(std::move(initial_metadata),
+            ->MakeCallPromise(std::move(call_args),
                               std::move(next_promise_factory));
       },
       // start_transport_op
@@ -382,7 +396,7 @@ MakePromiseBasedFilter(const char* name) {
       sizeof(CallData),
       // init_call_elem
       [](grpc_call_element* elem, const grpc_call_element_args* args) {
-        new (elem->call_data) CallData(elem, args);
+        new (elem->call_data) CallData(elem, args, kFlags);
         return GRPC_ERROR_NONE;
       },
       // set_pollset_or_pollset_set
@@ -401,8 +415,8 @@ MakePromiseBasedFilter(const char* name) {
       // init_channel_elem
       [](grpc_channel_element* elem, grpc_channel_element_args* args) {
         GPR_ASSERT(!args->is_last);
-        auto status = F::Create(args->channel_args,
-                                ChannelFilter::Args(args->channel_stack));
+        auto status = F::Create(ChannelArgs::FromC(args->channel_args),
+                                ChannelFilter::Args(args->channel_stack, elem));
         if (!status.ok()) return absl_status_to_grpc_error(status.status());
         new (elem->channel_data) F(std::move(*status));
         return GRPC_ERROR_NONE;

data/src/core/lib/compression/compression_internal.cc

@@ -194,13 +194,7 @@ CompressionAlgorithmSet CompressionAlgorithmSet::FromString(
 }
 
 uint32_t CompressionAlgorithmSet::ToLegacyBitmask() const {
-  uint32_t x = 0;
-  for (size_t i = 0; i < GRPC_COMPRESS_ALGORITHMS_COUNT; i++) {
-    if (set_.is_set(i)) {
-      x |= (1u << i);
-    }
-  }
-  return x;
+  return set_.ToInt<uint32_t>();
 }
 
 absl::optional<grpc_compression_algorithm>

data/src/core/lib/debug/stats_data.cc

@@ -41,8 +41,6 @@ const char* grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = {
     "pollset_kick_wakeup_fd",
     "pollset_kick_wakeup_cv",
     "pollset_kick_own_thread",
-    "syscall_epoll_ctl",
-    "pollset_fd_cache_hits",
     "histogram_slow_lookups",
     "syscall_write",
     "syscall_read",
@@ -75,6 +73,7 @@ const char* grpc_stats_counter_name[GRPC_STATS_COUNTER_COUNT] = {
     "http2_initiate_write_due_to_stream_flow_control",
     "http2_initiate_write_due_to_transport_flow_control",
     "http2_initiate_write_due_to_send_settings",
+    "http2_initiate_write_due_to_settings_ack",
     "http2_initiate_write_due_to_bdp_estimator_ping",
     "http2_initiate_write_due_to_flow_control_unstalled_by_setting",
     "http2_initiate_write_due_to_flow_control_unstalled_by_update",
@@ -147,9 +146,6 @@ const char* grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = {
     "polling wakeup (only valid for epoll1 right now)",
     "How many times could a polling wakeup be satisfied by keeping the waking "
     "thread awake? (only valid for epoll1 right now)",
-    "Number of epoll_ctl calls made (only valid for epollex right now)",
-    "Number of epoll_ctl calls skipped because the fd was cached as already "
-    "being added.  (only valid for epollex right now)",
     "Number of times histogram increments went through the slow (binary "
     "search) path",
     "Number of write syscalls (or equivalent - eg sendmsg) made by this "
@@ -186,6 +182,7 @@ const char* grpc_stats_counter_doc[GRPC_STATS_COUNTER_COUNT] = {
     "Number of HTTP2 writes initiated due to 'stream_flow_control'",
     "Number of HTTP2 writes initiated due to 'transport_flow_control'",
     "Number of HTTP2 writes initiated due to 'send_settings'",
+    "Number of HTTP2 writes initiated due to 'settings_ack'",
     "Number of HTTP2 writes initiated due to 'bdp_estimator_ping'",
     "Number of HTTP2 writes initiated due to "
     "'flow_control_unstalled_by_setting'",
@@ -281,7 +278,6 @@ const char* grpc_stats_histogram_doc[GRPC_STATS_HISTOGRAM_COUNT] = {
     "Number of streams whose payload was written per TCP write",
     "Number of streams terminated per TCP write",
     "Number of flow control updates written per TCP write",
-    // NOLINTNEXTLINE(bugprone-suspicious-missing-comma)
     "How many completion queues were checked looking for a CQ that had "
     "requested the incoming call",
 };

data/src/core/lib/debug/stats_data.h

@@ -42,8 +42,6 @@ typedef enum {
   GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_FD,
   GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_CV,
   GRPC_STATS_COUNTER_POLLSET_KICK_OWN_THREAD,
-  GRPC_STATS_COUNTER_SYSCALL_EPOLL_CTL,
-  GRPC_STATS_COUNTER_POLLSET_FD_CACHE_HITS,
   GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS,
   GRPC_STATS_COUNTER_SYSCALL_WRITE,
   GRPC_STATS_COUNTER_SYSCALL_READ,
@@ -76,6 +74,7 @@ typedef enum {
   GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_STREAM_FLOW_CONTROL,
   GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL,
   GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_SETTINGS,
+  GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SETTINGS_ACK,
   GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_BDP_ESTIMATOR_PING,
   GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_SETTING,
   GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_UPDATE,
@@ -206,10 +205,6 @@ typedef enum {
   GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_POLLSET_KICK_WAKEUP_CV)
 #define GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD() \
   GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_POLLSET_KICK_OWN_THREAD)
-#define GRPC_STATS_INC_SYSCALL_EPOLL_CTL() \
-  GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_SYSCALL_EPOLL_CTL)
-#define GRPC_STATS_INC_POLLSET_FD_CACHE_HITS() \
-  GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_POLLSET_FD_CACHE_HITS)
 #define GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS() \
   GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_HISTOGRAM_SLOW_LOOKUPS)
 #define GRPC_STATS_INC_SYSCALL_WRITE() \
@@ -287,6 +282,9 @@ typedef enum {
 #define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_SETTINGS() \
   GRPC_STATS_INC_COUNTER(                                          \
       GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SEND_SETTINGS)
+#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SETTINGS_ACK() \
+  GRPC_STATS_INC_COUNTER(                                         \
+      GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_SETTINGS_ACK)
 #define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_BDP_ESTIMATOR_PING() \
   GRPC_STATS_INC_COUNTER(                                               \
       GRPC_STATS_COUNTER_HTTP2_INITIATE_WRITE_DUE_TO_BDP_ESTIMATOR_PING)
@@ -398,43 +396,43 @@ typedef enum {
   GRPC_STATS_INC_COUNTER(GRPC_STATS_COUNTER_CQ_EV_QUEUE_TRANSIENT_POP_FAILURES)
 #define GRPC_STATS_INC_CALL_INITIAL_SIZE(value) \
   grpc_stats_inc_call_initial_size((int)(value))
-void grpc_stats_inc_call_initial_size(int value);
+void grpc_stats_inc_call_initial_size(int x);
 #define GRPC_STATS_INC_POLL_EVENTS_RETURNED(value) \
   grpc_stats_inc_poll_events_returned((int)(value))
-void grpc_stats_inc_poll_events_returned(int value);
+void grpc_stats_inc_poll_events_returned(int x);
 #define GRPC_STATS_INC_TCP_WRITE_SIZE(value) \
   grpc_stats_inc_tcp_write_size((int)(value))
-void grpc_stats_inc_tcp_write_size(int value);
+void grpc_stats_inc_tcp_write_size(int x);
 #define GRPC_STATS_INC_TCP_WRITE_IOV_SIZE(value) \
   grpc_stats_inc_tcp_write_iov_size((int)(value))
-void grpc_stats_inc_tcp_write_iov_size(int value);
+void grpc_stats_inc_tcp_write_iov_size(int x);
 #define GRPC_STATS_INC_TCP_READ_SIZE(value) \
   grpc_stats_inc_tcp_read_size((int)(value))
-void grpc_stats_inc_tcp_read_size(int value);
+void grpc_stats_inc_tcp_read_size(int x);
 #define GRPC_STATS_INC_TCP_READ_OFFER(value) \
   grpc_stats_inc_tcp_read_offer((int)(value))
-void grpc_stats_inc_tcp_read_offer(int value);
+void grpc_stats_inc_tcp_read_offer(int x);
 #define GRPC_STATS_INC_TCP_READ_OFFER_IOV_SIZE(value) \
   grpc_stats_inc_tcp_read_offer_iov_size((int)(value))
-void grpc_stats_inc_tcp_read_offer_iov_size(int value);
+void grpc_stats_inc_tcp_read_offer_iov_size(int x);
 #define GRPC_STATS_INC_HTTP2_SEND_MESSAGE_SIZE(value) \
   grpc_stats_inc_http2_send_message_size((int)(value))
-void grpc_stats_inc_http2_send_message_size(int value);
+void grpc_stats_inc_http2_send_message_size(int x);
 #define GRPC_STATS_INC_HTTP2_SEND_INITIAL_METADATA_PER_WRITE(value) \
   grpc_stats_inc_http2_send_initial_metadata_per_write((int)(value))
-void grpc_stats_inc_http2_send_initial_metadata_per_write(int value);
+void grpc_stats_inc_http2_send_initial_metadata_per_write(int x);
 #define GRPC_STATS_INC_HTTP2_SEND_MESSAGE_PER_WRITE(value) \
   grpc_stats_inc_http2_send_message_per_write((int)(value))
-void grpc_stats_inc_http2_send_message_per_write(int value);
+void grpc_stats_inc_http2_send_message_per_write(int x);
 #define GRPC_STATS_INC_HTTP2_SEND_TRAILING_METADATA_PER_WRITE(value) \
   grpc_stats_inc_http2_send_trailing_metadata_per_write((int)(value))
-void grpc_stats_inc_http2_send_trailing_metadata_per_write(int value);
+void grpc_stats_inc_http2_send_trailing_metadata_per_write(int x);
 #define GRPC_STATS_INC_HTTP2_SEND_FLOWCTL_PER_WRITE(value) \
   grpc_stats_inc_http2_send_flowctl_per_write((int)(value))
-void grpc_stats_inc_http2_send_flowctl_per_write(int value);
+void grpc_stats_inc_http2_send_flowctl_per_write(int x);
 #define GRPC_STATS_INC_SERVER_CQS_CHECKED(value) \
   grpc_stats_inc_server_cqs_checked((int)(value))
-void grpc_stats_inc_server_cqs_checked(int value);
+void grpc_stats_inc_server_cqs_checked(int x);
 #else
 #define GRPC_STATS_INC_CLIENT_CALLS_CREATED()
 #define GRPC_STATS_INC_SERVER_CALLS_CREATED()
@@ -450,8 +448,6 @@ void grpc_stats_inc_server_cqs_checked(int value);
 #define GRPC_STATS_INC_POLLSET_KICK_WAKEUP_FD()
 #define GRPC_STATS_INC_POLLSET_KICK_WAKEUP_CV()
 #define GRPC_STATS_INC_POLLSET_KICK_OWN_THREAD()
-#define GRPC_STATS_INC_SYSCALL_EPOLL_CTL()
-#define GRPC_STATS_INC_POLLSET_FD_CACHE_HITS()
 #define GRPC_STATS_INC_HISTOGRAM_SLOW_LOOKUPS()
 #define GRPC_STATS_INC_SYSCALL_WRITE()
 #define GRPC_STATS_INC_SYSCALL_READ()
@@ -484,6 +480,7 @@ void grpc_stats_inc_server_cqs_checked(int value);
 #define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_STREAM_FLOW_CONTROL()
 #define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_TRANSPORT_FLOW_CONTROL()
 #define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SEND_SETTINGS()
+#define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_SETTINGS_ACK()
 #define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_BDP_ESTIMATOR_PING()
 #define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_SETTING()
 #define GRPC_STATS_INC_HTTP2_INITIATE_WRITE_DUE_TO_FLOW_CONTROL_UNSTALLED_BY_UPDATE()

data/src/core/lib/gpr/tls.h

@@ -50,6 +50,7 @@ class TlsTypeConstrainer {
 
 #include <pthread.h>
 
+#include <algorithm>
 #include <array>
 #include <cstring>
 

data/src/core/lib/gprpp/bitset.h

@@ -153,6 +153,18 @@ class BitSet {
     return true;
   }
 
+  template <typename Int>
+  typename std::enable_if<std::is_unsigned<Int>::value &&
+                              (sizeof(Int) * 8 >= kTotalBits),
+                          Int>::type
+  ToInt() const {
+    Int result = 0;
+    for (size_t i = 0; i < kTotalBits; i++) {
+      if (is_set(i)) result |= (Int(1) << i);
+    }
+    return result;
+  }
+
  private:
   // Given a bit index, return which unit it's stored in.
   static constexpr size_t unit_for(size_t bit) { return bit / kUnitBits; }

data/src/core/lib/gprpp/cpp_impl_of.h

@@ -31,6 +31,10 @@ class CppImplOf {
     return reinterpret_cast<CppType*>(c_type);
   }
 
+  static const CppType* FromC(const CType* c_type) {
+    return reinterpret_cast<const CppType*>(c_type);
+  }
+
   // Retrieve a c pointer (of the same ownership as this)
   CType* c_ptr() {
     return reinterpret_cast<CType*>(static_cast<CppType*>(this));

data/src/core/lib/gprpp/match.h

@@ -0,0 +1,73 @@
+// Copyright 2021 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_CORE_LIB_GPRPP_MATCH_H
+#define GRPC_CORE_LIB_GPRPP_MATCH_H
+
+#include <grpc/support/port_platform.h>
+
+#include "absl/types/variant.h"
+
+#include "src/core/lib/gprpp/overload.h"
+
+namespace grpc_core {
+
+namespace detail {
+
+template <typename... Cases>
+struct MatchPointerExtractor {
+  OverloadType<Cases...> cases;
+  template <typename T>
+  auto operator()(T& value) -> decltype(cases(&value)) {
+    return cases(&value);
+  }
+};
+
+}  // namespace detail
+
+/// Match on a variant.
+/// Given variant \a value, and a set of callables \a fs, call the appropriate
+/// callable based on the type contained in \a value.
+///
+/// Example (prints "hoorah"):
+///   variant<int, string> v = 42;
+///   Match(v,
+///         [](int i) { puts("hoorah"); },
+///         [](string s) { puts("boo"); });
+template <typename... Fs, typename T0, typename... Ts>
+auto Match(const absl::variant<T0, Ts...>& value, Fs... fs)
+    -> decltype(std::declval<OverloadType<Fs...>>()(std::declval<T0>())) {
+  return absl::visit(Overload(std::move(fs)...), value);
+}
+
+/// A version of Match that takes a mutable pointer to a variant and calls its
+/// overload callables with a mutable pointer to the current variant value.
+///
+/// Example:
+///   variant<int, string> v = 42;
+///   MatchMutable(&v,
+///                [](int* i) { *i = 1; },
+///                [](string* s) { *s = "foo"; });
+///   // v now contains 1.
+template <typename... Fs, typename T0, typename... Ts>
+auto MatchMutable(absl::variant<T0, Ts...>* value, Fs... fs)
+    -> decltype(std::declval<OverloadType<Fs...>>()(std::declval<T0*>())) {
+  return absl::visit(detail::MatchPointerExtractor<Fs...>{OverloadType<Fs...>(
+                         std::move(fs)...)},
+                     *value);
+}
+
+}  // namespace grpc_core
+
+#endif  // GRPC_CORE_LIB_GPRPP_MATCH_H

data/src/core/lib/gprpp/overload.h

@@ -0,0 +1,59 @@
+// Copyright 2021 gRPC authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GRPC_CORE_LIB_GPRPP_OVERLOAD_H
+#define GRPC_CORE_LIB_GPRPP_OVERLOAD_H
+
+#include <grpc/support/port_platform.h>
+
+#include <utility>
+
+namespace grpc_core {
+
+template <typename... Cases>
+struct OverloadType;
+// Compose one overload with N more -- use inheritance to leverage using and the
+// empty base class optimization.
+template <typename Case, typename... Cases>
+struct OverloadType<Case, Cases...> : public Case,
+                                      public OverloadType<Cases...> {
+  explicit OverloadType(Case&& c, Cases&&... cases)
+      : Case(std::forward<Case>(c)),
+        OverloadType<Cases...>(std::forward<Cases>(cases)...) {}
+  using Case::operator();
+  using OverloadType<Cases...>::operator();
+};
+// Overload of a single case is just that case itself
+template <typename Case>
+struct OverloadType<Case> : public Case {
+  explicit OverloadType(Case&& c) : Case(std::forward<Case>(c)) {}
+  using Case::operator();
+};
+
+/// Compose callables into a single callable.
+/// e.g. given [](int i) { puts("a"); } and [](double d) { puts("b"); },
+/// return a callable object like:
+/// struct {
+///   void operator()(int i) { puts("a"); }
+///   void operator()(double i) { puts("b"); }
+/// };
+/// Preserves all captures.
+template <typename... Cases>
+OverloadType<Cases...> Overload(Cases... cases) {
+  return OverloadType<Cases...>(std::move(cases)...);
+}
+
+}  // namespace grpc_core
+
+#endif  // GRPC_CORE_LIB_GPRPP_OVERLOAD_H

data/src/core/lib/gprpp/ref_counted.h

@@ -281,6 +281,8 @@ template <typename Child, typename Impl = PolymorphicRefCount,
           UnrefBehavior UnrefBehaviorArg = kUnrefDelete>
 class RefCounted : public Impl {
  public:
+  using RefCountedChildType = Child;
+
   // Note: Depending on the Impl used, this dtor can be implicitly virtual.
   ~RefCounted() = default;