grpc 1.58.0 → 1.59.0

data/src/core/lib/experiments/experiments.h

@@ -50,7 +50,6 @@
 
 #include <grpc/support/port_platform.h>
 
-#include <stddef.h>
 #include "src/core/lib/experiments/config.h"
 
 namespace grpc_core {
@@ -69,19 +68,28 @@ inline bool IsMonitoringExperimentEnabled() { return true; }
 inline bool IsPromiseBasedClientCallEnabled() { return false; }
 inline bool IsFreeLargeAllocatorEnabled() { return false; }
 inline bool IsPromiseBasedServerCallEnabled() { return false; }
-inline bool IsTransportSuppliesClientLatencyEnabled() { return false; }
 inline bool IsEventEngineListenerEnabled() { return false; }
 inline bool IsScheduleCancellationOverWriteEnabled() { return false; }
 inline bool IsTraceRecordCallopsEnabled() { return false; }
 inline bool IsEventEngineDnsEnabled() { return false; }
-inline bool IsWorkStealingEnabled() { return false; }
 inline bool IsClientPrivacyEnabled() { return false; }
 inline bool IsCanaryClientPrivacyEnabled() { return false; }
 inline bool IsServerPrivacyEnabled() { return false; }
-#define GRPC_EXPERIMENT_IS_INCLUDED_UNIQUE_METADATA_STRINGS
-inline bool IsUniqueMetadataStringsEnabled() { return true; }
 inline bool IsKeepaliveFixEnabled() { return false; }
 inline bool IsKeepaliveServerFixEnabled() { return false; }
+inline bool IsWorkSerializerDispatchEnabled() { return false; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_LAZIER_STREAM_UPDATES
+inline bool IsLazierStreamUpdatesEnabled() { return true; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_JITTER_MAX_IDLE
+inline bool IsJitterMaxIdleEnabled() { return true; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_ROUND_ROBIN_DELEGATE_TO_PICK_FIRST
+inline bool IsRoundRobinDelegateToPickFirstEnabled() { return true; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_WRR_DELEGATE_TO_PICK_FIRST
+inline bool IsWrrDelegateToPickFirstEnabled() { return true; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_COMBINER_OFFLOAD_TO_EVENT_ENGINE
+inline bool IsCombinerOffloadToEventEngineEnabled() { return true; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_REGISTERED_METHOD_LOOKUP_IN_TRANSPORT
+inline bool IsRegisteredMethodLookupInTransportEnabled() { return true; }
 
 #elif defined(GPR_WINDOWS)
 inline bool IsTcpFrameSizeTuningEnabled() { return false; }
@@ -95,19 +103,28 @@ inline bool IsMonitoringExperimentEnabled() { return true; }
 inline bool IsPromiseBasedClientCallEnabled() { return false; }
 inline bool IsFreeLargeAllocatorEnabled() { return false; }
 inline bool IsPromiseBasedServerCallEnabled() { return false; }
-inline bool IsTransportSuppliesClientLatencyEnabled() { return false; }
 inline bool IsEventEngineListenerEnabled() { return false; }
 inline bool IsScheduleCancellationOverWriteEnabled() { return false; }
 inline bool IsTraceRecordCallopsEnabled() { return false; }
 inline bool IsEventEngineDnsEnabled() { return false; }
-inline bool IsWorkStealingEnabled() { return false; }
 inline bool IsClientPrivacyEnabled() { return false; }
 inline bool IsCanaryClientPrivacyEnabled() { return false; }
 inline bool IsServerPrivacyEnabled() { return false; }
-#define GRPC_EXPERIMENT_IS_INCLUDED_UNIQUE_METADATA_STRINGS
-inline bool IsUniqueMetadataStringsEnabled() { return true; }
 inline bool IsKeepaliveFixEnabled() { return false; }
 inline bool IsKeepaliveServerFixEnabled() { return false; }
+inline bool IsWorkSerializerDispatchEnabled() { return false; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_LAZIER_STREAM_UPDATES
+inline bool IsLazierStreamUpdatesEnabled() { return true; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_JITTER_MAX_IDLE
+inline bool IsJitterMaxIdleEnabled() { return true; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_ROUND_ROBIN_DELEGATE_TO_PICK_FIRST
+inline bool IsRoundRobinDelegateToPickFirstEnabled() { return true; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_WRR_DELEGATE_TO_PICK_FIRST
+inline bool IsWrrDelegateToPickFirstEnabled() { return true; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_COMBINER_OFFLOAD_TO_EVENT_ENGINE
+inline bool IsCombinerOffloadToEventEngineEnabled() { return true; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_REGISTERED_METHOD_LOOKUP_IN_TRANSPORT
+inline bool IsRegisteredMethodLookupInTransportEnabled() { return true; }
 
 #else
 inline bool IsTcpFrameSizeTuningEnabled() { return false; }
@@ -121,68 +138,165 @@ inline bool IsMonitoringExperimentEnabled() { return true; }
 inline bool IsPromiseBasedClientCallEnabled() { return false; }
 inline bool IsFreeLargeAllocatorEnabled() { return false; }
 inline bool IsPromiseBasedServerCallEnabled() { return false; }
-inline bool IsTransportSuppliesClientLatencyEnabled() { return false; }
 inline bool IsEventEngineListenerEnabled() { return false; }
 inline bool IsScheduleCancellationOverWriteEnabled() { return false; }
 inline bool IsTraceRecordCallopsEnabled() { return false; }
 inline bool IsEventEngineDnsEnabled() { return false; }
-inline bool IsWorkStealingEnabled() { return false; }
 inline bool IsClientPrivacyEnabled() { return false; }
 inline bool IsCanaryClientPrivacyEnabled() { return false; }
 inline bool IsServerPrivacyEnabled() { return false; }
-#define GRPC_EXPERIMENT_IS_INCLUDED_UNIQUE_METADATA_STRINGS
-inline bool IsUniqueMetadataStringsEnabled() { return true; }
 inline bool IsKeepaliveFixEnabled() { return false; }
 inline bool IsKeepaliveServerFixEnabled() { return false; }
+inline bool IsWorkSerializerDispatchEnabled() { return false; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_LAZIER_STREAM_UPDATES
+inline bool IsLazierStreamUpdatesEnabled() { return true; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_JITTER_MAX_IDLE
+inline bool IsJitterMaxIdleEnabled() { return true; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_ROUND_ROBIN_DELEGATE_TO_PICK_FIRST
+inline bool IsRoundRobinDelegateToPickFirstEnabled() { return true; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_WRR_DELEGATE_TO_PICK_FIRST
+inline bool IsWrrDelegateToPickFirstEnabled() { return true; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_COMBINER_OFFLOAD_TO_EVENT_ENGINE
+inline bool IsCombinerOffloadToEventEngineEnabled() { return true; }
+#define GRPC_EXPERIMENT_IS_INCLUDED_REGISTERED_METHOD_LOOKUP_IN_TRANSPORT
+inline bool IsRegisteredMethodLookupInTransportEnabled() { return true; }
 #endif
 
 #else
+enum ExperimentIds {
+  kExperimentIdTcpFrameSizeTuning,
+  kExperimentIdTcpRcvLowat,
+  kExperimentIdPeerStateBasedFraming,
+  kExperimentIdMemoryPressureController,
+  kExperimentIdUnconstrainedMaxQuotaBufferSize,
+  kExperimentIdEventEngineClient,
+  kExperimentIdMonitoringExperiment,
+  kExperimentIdPromiseBasedClientCall,
+  kExperimentIdFreeLargeAllocator,
+  kExperimentIdPromiseBasedServerCall,
+  kExperimentIdEventEngineListener,
+  kExperimentIdScheduleCancellationOverWrite,
+  kExperimentIdTraceRecordCallops,
+  kExperimentIdEventEngineDns,
+  kExperimentIdClientPrivacy,
+  kExperimentIdCanaryClientPrivacy,
+  kExperimentIdServerPrivacy,
+  kExperimentIdKeepaliveFix,
+  kExperimentIdKeepaliveServerFix,
+  kExperimentIdWorkSerializerDispatch,
+  kExperimentIdLazierStreamUpdates,
+  kExperimentIdJitterMaxIdle,
+  kExperimentIdRoundRobinDelegateToPickFirst,
+  kExperimentIdWrrDelegateToPickFirst,
+  kExperimentIdCombinerOffloadToEventEngine,
+  kExperimentIdRegisteredMethodLookupInTransport,
+  kNumExperiments
+};
 #define GRPC_EXPERIMENT_IS_INCLUDED_TCP_FRAME_SIZE_TUNING
-inline bool IsTcpFrameSizeTuningEnabled() { return IsExperimentEnabled(0); }
+inline bool IsTcpFrameSizeTuningEnabled() {
+  return IsExperimentEnabled(kExperimentIdTcpFrameSizeTuning);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_TCP_RCV_LOWAT
-inline bool IsTcpRcvLowatEnabled() { return IsExperimentEnabled(1); }
+inline bool IsTcpRcvLowatEnabled() {
+  return IsExperimentEnabled(kExperimentIdTcpRcvLowat);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_PEER_STATE_BASED_FRAMING
-inline bool IsPeerStateBasedFramingEnabled() { return IsExperimentEnabled(2); }
+inline bool IsPeerStateBasedFramingEnabled() {
+  return IsExperimentEnabled(kExperimentIdPeerStateBasedFraming);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_MEMORY_PRESSURE_CONTROLLER
-inline bool IsMemoryPressureControllerEnabled() { return IsExperimentEnabled(3); }
+inline bool IsMemoryPressureControllerEnabled() {
+  return IsExperimentEnabled(kExperimentIdMemoryPressureController);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_UNCONSTRAINED_MAX_QUOTA_BUFFER_SIZE
-inline bool IsUnconstrainedMaxQuotaBufferSizeEnabled() { return IsExperimentEnabled(4); }
+inline bool IsUnconstrainedMaxQuotaBufferSizeEnabled() {
+  return IsExperimentEnabled(kExperimentIdUnconstrainedMaxQuotaBufferSize);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_EVENT_ENGINE_CLIENT
-inline bool IsEventEngineClientEnabled() { return IsExperimentEnabled(5); }
+inline bool IsEventEngineClientEnabled() {
+  return IsExperimentEnabled(kExperimentIdEventEngineClient);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_MONITORING_EXPERIMENT
-inline bool IsMonitoringExperimentEnabled() { return IsExperimentEnabled(6); }
+inline bool IsMonitoringExperimentEnabled() {
+  return IsExperimentEnabled(kExperimentIdMonitoringExperiment);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_CLIENT_CALL
-inline bool IsPromiseBasedClientCallEnabled() { return IsExperimentEnabled(7); }
+inline bool IsPromiseBasedClientCallEnabled() {
+  return IsExperimentEnabled(kExperimentIdPromiseBasedClientCall);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_FREE_LARGE_ALLOCATOR
-inline bool IsFreeLargeAllocatorEnabled() { return IsExperimentEnabled(8); }
+inline bool IsFreeLargeAllocatorEnabled() {
+  return IsExperimentEnabled(kExperimentIdFreeLargeAllocator);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_PROMISE_BASED_SERVER_CALL
-inline bool IsPromiseBasedServerCallEnabled() { return IsExperimentEnabled(9); }
-#define GRPC_EXPERIMENT_IS_INCLUDED_TRANSPORT_SUPPLIES_CLIENT_LATENCY
-inline bool IsTransportSuppliesClientLatencyEnabled() { return IsExperimentEnabled(10); }
+inline bool IsPromiseBasedServerCallEnabled() {
+  return IsExperimentEnabled(kExperimentIdPromiseBasedServerCall);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_EVENT_ENGINE_LISTENER
-inline bool IsEventEngineListenerEnabled() { return IsExperimentEnabled(11); }
+inline bool IsEventEngineListenerEnabled() {
+  return IsExperimentEnabled(kExperimentIdEventEngineListener);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_SCHEDULE_CANCELLATION_OVER_WRITE
-inline bool IsScheduleCancellationOverWriteEnabled() { return IsExperimentEnabled(12); }
+inline bool IsScheduleCancellationOverWriteEnabled() {
+  return IsExperimentEnabled(kExperimentIdScheduleCancellationOverWrite);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_TRACE_RECORD_CALLOPS
-inline bool IsTraceRecordCallopsEnabled() { return IsExperimentEnabled(13); }
+inline bool IsTraceRecordCallopsEnabled() {
+  return IsExperimentEnabled(kExperimentIdTraceRecordCallops);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_EVENT_ENGINE_DNS
-inline bool IsEventEngineDnsEnabled() { return IsExperimentEnabled(14); }
-#define GRPC_EXPERIMENT_IS_INCLUDED_WORK_STEALING
-inline bool IsWorkStealingEnabled() { return IsExperimentEnabled(15); }
+inline bool IsEventEngineDnsEnabled() {
+  return IsExperimentEnabled(kExperimentIdEventEngineDns);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_CLIENT_PRIVACY
-inline bool IsClientPrivacyEnabled() { return IsExperimentEnabled(16); }
+inline bool IsClientPrivacyEnabled() {
+  return IsExperimentEnabled(kExperimentIdClientPrivacy);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_CANARY_CLIENT_PRIVACY
-inline bool IsCanaryClientPrivacyEnabled() { return IsExperimentEnabled(17); }
+inline bool IsCanaryClientPrivacyEnabled() {
+  return IsExperimentEnabled(kExperimentIdCanaryClientPrivacy);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_SERVER_PRIVACY
-inline bool IsServerPrivacyEnabled() { return IsExperimentEnabled(18); }
-#define GRPC_EXPERIMENT_IS_INCLUDED_UNIQUE_METADATA_STRINGS
-inline bool IsUniqueMetadataStringsEnabled() { return IsExperimentEnabled(19); }
+inline bool IsServerPrivacyEnabled() {
+  return IsExperimentEnabled(kExperimentIdServerPrivacy);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_KEEPALIVE_FIX
-inline bool IsKeepaliveFixEnabled() { return IsExperimentEnabled(20); }
+inline bool IsKeepaliveFixEnabled() {
+  return IsExperimentEnabled(kExperimentIdKeepaliveFix);
+}
 #define GRPC_EXPERIMENT_IS_INCLUDED_KEEPALIVE_SERVER_FIX
-inline bool IsKeepaliveServerFixEnabled() { return IsExperimentEnabled(21); }
+inline bool IsKeepaliveServerFixEnabled() {
+  return IsExperimentEnabled(kExperimentIdKeepaliveServerFix);
+}
+#define GRPC_EXPERIMENT_IS_INCLUDED_WORK_SERIALIZER_DISPATCH
+inline bool IsWorkSerializerDispatchEnabled() {
+  return IsExperimentEnabled(kExperimentIdWorkSerializerDispatch);
+}
+#define GRPC_EXPERIMENT_IS_INCLUDED_LAZIER_STREAM_UPDATES
+inline bool IsLazierStreamUpdatesEnabled() {
+  return IsExperimentEnabled(kExperimentIdLazierStreamUpdates);
+}
+#define GRPC_EXPERIMENT_IS_INCLUDED_JITTER_MAX_IDLE
+inline bool IsJitterMaxIdleEnabled() {
+  return IsExperimentEnabled(kExperimentIdJitterMaxIdle);
+}
+#define GRPC_EXPERIMENT_IS_INCLUDED_ROUND_ROBIN_DELEGATE_TO_PICK_FIRST
+inline bool IsRoundRobinDelegateToPickFirstEnabled() {
+  return IsExperimentEnabled(kExperimentIdRoundRobinDelegateToPickFirst);
+}
+#define GRPC_EXPERIMENT_IS_INCLUDED_WRR_DELEGATE_TO_PICK_FIRST
+inline bool IsWrrDelegateToPickFirstEnabled() {
+  return IsExperimentEnabled(kExperimentIdWrrDelegateToPickFirst);
+}
+#define GRPC_EXPERIMENT_IS_INCLUDED_COMBINER_OFFLOAD_TO_EVENT_ENGINE
+inline bool IsCombinerOffloadToEventEngineEnabled() {
+  return IsExperimentEnabled(kExperimentIdCombinerOffloadToEventEngine);
+}
+#define GRPC_EXPERIMENT_IS_INCLUDED_REGISTERED_METHOD_LOOKUP_IN_TRANSPORT
+inline bool IsRegisteredMethodLookupInTransportEnabled() {
+  return IsExperimentEnabled(kExperimentIdRegisteredMethodLookupInTransport);
+}
 
-constexpr const size_t kNumExperiments = 22;
 extern const ExperimentMetadata g_experiment_metadata[kNumExperiments];
 
 #endif

data/src/core/lib/gprpp/fork.h

@@ -31,7 +31,7 @@
 
 namespace grpc_core {
 
-class Fork {
+class GPR_DLL Fork {
  public:
   typedef void (*child_postfork_func)(void);
 

data/src/core/lib/gprpp/manual_constructor.h

@@ -25,7 +25,6 @@
 
 #include <stddef.h>
 
-#include <type_traits>
 #include <utility>
 
 #include "src/core/lib/gprpp/construct_destruct.h"
@@ -139,7 +138,7 @@ class ManualConstructor {
   void Destroy() { Destruct(get()); }
 
  private:
-  typename std::aligned_storage<sizeof(Type), alignof(Type)>::type space_;
+  alignas(Type) char space_[sizeof(Type)];
 };
 
 }  // namespace grpc_core

data/src/core/lib/gprpp/no_destruct.h

@@ -68,7 +68,7 @@ class NoDestruct {
   const T* get() const { return reinterpret_cast<const T*>(&space_); }
 
  private:
-  typename std::aligned_storage<sizeof(T), alignof(T)>::type space_;
+  alignas(T) char space_[sizeof(T)];
 };
 
 // Helper for when a program desires a single *process wide* instance of a

data/src/core/lib/gprpp/per_cpu.cc

@@ -22,6 +22,8 @@
 
 namespace grpc_core {
 
+thread_local PerCpuShardingHelper::State PerCpuShardingHelper::state_;
+
 size_t PerCpuOptions::Shards() {
   return ShardsForCpuCount(gpr_cpu_num_cores());
 }

data/src/core/lib/gprpp/per_cpu.h

@@ -17,12 +17,18 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <stdint.h>
+
 #include <algorithm>
 #include <cstddef>
 #include <limits>
 #include <memory>
 
-#include "src/core/lib/iomgr/exec_ctx.h"
+#include <grpc/support/cpu.h>
+
+// Sharded collections of objects
+// This used to be per-cpu, now it's much less so - but still a way to limit
+// contention.
 
 namespace grpc_core {
 
@@ -51,23 +57,46 @@ class PerCpuOptions {
   size_t max_shards_ = std::numeric_limits<size_t>::max();
 };
 
+class PerCpuShardingHelper {
+ public:
+  size_t GetShardingBits() {
+    // We periodically refresh the last seen cpu to try to ensure that we spread
+    // load evenly over all shards of a per-cpu data structure, even in the
+    // event of shifting thread distributions, load patterns.
+    // Ideally we'd just call gpr_cpu_current_cpu() every call of this function
+    // to get perfect distribution, but that function is currently quite slow on
+    // some platforms and so we need to cache it somewhat.
+    if (GPR_UNLIKELY(state_.uses_until_refresh == 0)) state_ = State();
+    --state_.uses_until_refresh;
+    return state_.last_seen_cpu;
+  }
+
+ private:
+  struct State {
+    uint16_t last_seen_cpu = gpr_cpu_current_cpu();
+    uint16_t uses_until_refresh = 65535;
+  };
+  static thread_local State state_;
+};
+
 template <typename T>
 class PerCpu {
  public:
   // Options are not defaulted to try and force consideration of what the
   // options specify.
-  explicit PerCpu(PerCpuOptions options) : cpus_(options.Shards()) {}
+  explicit PerCpu(PerCpuOptions options) : shards_(options.Shards()) {}
 
-  T& this_cpu() { return data_[ExecCtx::Get()->starting_cpu() % cpus_]; }
+  T& this_cpu() { return data_[sharding_helper_.GetShardingBits() % shards_]; }
 
   T* begin() { return data_.get(); }
-  T* end() { return data_.get() + cpus_; }
+  T* end() { return data_.get() + shards_; }
   const T* begin() const { return data_.get(); }
-  const T* end() const { return data_.get() + cpus_; }
+  const T* end() const { return data_.get() + shards_; }
 
  private:
-  const size_t cpus_;
-  std::unique_ptr<T[]> data_{new T[cpus_]};
+  PerCpuShardingHelper sharding_helper_;
+  const size_t shards_;
+  std::unique_ptr<T[]> data_{new T[shards_]};
 };
 
 }  // namespace grpc_core

data/src/core/lib/gprpp/ref_counted_ptr.h

@@ -21,10 +21,14 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <stddef.h>
+
 #include <iosfwd>
 #include <type_traits>
 #include <utility>
 
+#include "absl/hash/hash.h"
+
 #include "src/core/lib/gprpp/debug_location.h"
 
 namespace grpc_core {
@@ -333,6 +337,65 @@ bool operator<(const WeakRefCountedPtr<T>& p1, const WeakRefCountedPtr<T>& p2) {
   return p1.get() < p2.get();
 }
 
+//
+// absl::Hash integration
+//
+
+template <typename H, typename T>
+H AbslHashValue(H h, const RefCountedPtr<T>& p) {
+  return H::combine(std::move(h), p.get());
+}
+
+template <typename H, typename T>
+H AbslHashValue(H h, const WeakRefCountedPtr<T>& p) {
+  return H::combine(std::move(h), p.get());
+}
+
+// Heterogenous lookup support.
+template <typename T>
+struct RefCountedPtrHash {
+  using is_transparent = void;
+  size_t operator()(const RefCountedPtr<T>& p) const {
+    return absl::Hash<RefCountedPtr<T>>{}(p);
+  }
+  size_t operator()(const WeakRefCountedPtr<T>& p) const {
+    return absl::Hash<WeakRefCountedPtr<T>>{}(p);
+  }
+  size_t operator()(T* p) const { return absl::Hash<T*>{}(p); }
+};
+template <typename T>
+struct RefCountedPtrEq {
+  using is_transparent = void;
+  bool operator()(const RefCountedPtr<T>& p1,
+                  const RefCountedPtr<T>& p2) const {
+    return p1 == p2;
+  }
+  bool operator()(const WeakRefCountedPtr<T>& p1,
+                  const WeakRefCountedPtr<T>& p2) const {
+    return p1 == p2;
+  }
+  bool operator()(const RefCountedPtr<T>& p1,
+                  const WeakRefCountedPtr<T>& p2) const {
+    return p1 == p2.get();
+  }
+  bool operator()(const WeakRefCountedPtr<T>& p1,
+                  const RefCountedPtr<T>& p2) const {
+    return p1 == p2.get();
+  }
+  bool operator()(const RefCountedPtr<T>& p1, const T* p2) const {
+    return p1 == p2;
+  }
+  bool operator()(const WeakRefCountedPtr<T>& p1, const T* p2) const {
+    return p1 == p2;
+  }
+  bool operator()(const T* p1, const RefCountedPtr<T>& p2) const {
+    return p2 == p1;
+  }
+  bool operator()(const T* p1, const WeakRefCountedPtr<T>& p2) const {
+    return p2 == p1;
+  }
+};
+
 }  // namespace grpc_core
 
 #endif  // GRPC_SRC_CORE_LIB_GPRPP_REF_COUNTED_PTR_H