grpc 1.63.0 → 1.66.0

data/src/core/lib/resource_quota/arena.h

@@ -25,8 +25,6 @@
 #ifndef GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_ARENA_H
 #define GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_ARENA_H
 
-#include <grpc/support/port_platform.h>
-
 #include <stddef.h>
 
 #include <atomic>
@@ -35,82 +33,75 @@
 #include <utility>
 
 #include <grpc/event_engine/memory_allocator.h>
+#include <grpc/support/port_platform.h>
 
-#include "src/core/lib/gpr/alloc.h"
 #include "src/core/lib/gprpp/construct_destruct.h"
 #include "src/core/lib/promise/context.h"
 #include "src/core/lib/resource_quota/memory_quota.h"
-
-#define GRPC_ARENA_POOLED_ALLOCATIONS_USE_MALLOC
-// #define GRPC_ARENA_TRACE_POOLED_ALLOCATIONS
+#include "src/core/util/alloc.h"
 
 namespace grpc_core {
 
-namespace arena_detail {
+class Arena;
 
-#ifndef GRPC_ARENA_POOLED_ALLOCATIONS_USE_MALLOC
-struct PoolAndSize {
-  size_t alloc_size;
-  size_t pool_index;
-};
+template <typename T>
+struct ArenaContextType;
 
-template <typename Void, size_t kIndex, size_t kObjectSize,
-          size_t... kBucketSize>
-struct PoolIndexForSize;
-
-template <size_t kObjectSize, size_t kIndex, size_t kSmallestRemainingBucket,
-          size_t... kBucketSizes>
-struct PoolIndexForSize<
-    absl::enable_if_t<kObjectSize <= kSmallestRemainingBucket>, kIndex,
-    kObjectSize, kSmallestRemainingBucket, kBucketSizes...> {
-  static constexpr size_t kPool = kIndex;
-  static constexpr size_t kSize = kSmallestRemainingBucket;
-};
+namespace arena_detail {
 
-template <size_t kObjectSize, size_t kIndex, size_t kSmallestRemainingBucket,
-          size_t... kBucketSizes>
-struct PoolIndexForSize<
-    absl::enable_if_t<(kObjectSize > kSmallestRemainingBucket)>, kIndex,
-    kObjectSize, kSmallestRemainingBucket, kBucketSizes...>
-    : public PoolIndexForSize<void, kIndex + 1, kObjectSize, kBucketSizes...> {
-};
+// Tracks all registered arena context types (these should only be registered
+// via ArenaContextTraits at static initialization time).
+class BaseArenaContextTraits {
+ public:
+  // Count of number of contexts that have been allocated.
+  static uint16_t NumContexts() {
+    return static_cast<uint16_t>(RegisteredTraits().size());
+  }
 
-template <size_t kObjectSize, size_t... kBucketSizes>
-constexpr size_t PoolFromObjectSize(
-    absl::integer_sequence<size_t, kBucketSizes...>) {
-  return PoolIndexForSize<void, 0, kObjectSize, kBucketSizes...>::kPool;
-}
+  // Number of bytes required to store the context pointers on an arena.
+  static size_t ContextSize() { return NumContexts() * sizeof(void*); }
 
-template <size_t kObjectSize, size_t... kBucketSizes>
-constexpr size_t AllocationSizeFromObjectSize(
-    absl::integer_sequence<size_t, kBucketSizes...>) {
-  return PoolIndexForSize<void, 0, kObjectSize, kBucketSizes...>::kSize;
-}
+  // Call the registered destruction function for a context.
+  static void Destroy(uint16_t id, void* ptr) {
+    if (ptr == nullptr) return;
+    RegisteredTraits()[id](ptr);
+  }
 
-template <size_t kIndex, size_t... kBucketSizes>
-struct ChoosePoolForAllocationSizeImpl;
+ protected:
+  // Allocate a new context id and register the destruction function.
+  static uint16_t MakeId(void (*destroy)(void* ptr)) {
+    auto& traits = RegisteredTraits();
+    const uint16_t id = static_cast<uint16_t>(traits.size());
+    traits.push_back(destroy);
+    return id;
+  }
 
-template <size_t kIndex, size_t kBucketSize, size_t... kBucketSizes>
-struct ChoosePoolForAllocationSizeImpl<kIndex, kBucketSize, kBucketSizes...> {
-  static PoolAndSize Fn(size_t n) {
-    if (n <= kBucketSize) return {kBucketSize, kIndex};
-    return ChoosePoolForAllocationSizeImpl<kIndex + 1, kBucketSizes...>::Fn(n);
+ private:
+  static std::vector<void (*)(void*)>& RegisteredTraits() {
+    static NoDestruct<std::vector<void (*)(void*)>> registered_traits;
+    return *registered_traits;
   }
 };
 
-template <size_t kIndex>
-struct ChoosePoolForAllocationSizeImpl<kIndex> {
-  static PoolAndSize Fn(size_t n) {
-    return PoolAndSize{n, std::numeric_limits<size_t>::max()};
-  }
+// Traits for a specific context type.
+template <typename T>
+class ArenaContextTraits : public BaseArenaContextTraits {
+ public:
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static uint16_t id() { return id_; }
+
+ private:
+  static const uint16_t id_;
 };
 
-template <size_t... kBucketSizes>
-PoolAndSize ChoosePoolForAllocationSize(
-    size_t n, absl::integer_sequence<size_t, kBucketSizes...>) {
-  return ChoosePoolForAllocationSizeImpl<0, kBucketSizes...>::Fn(n);
+template <typename T>
+GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION void DestroyArenaContext(void* p) {
+  ArenaContextType<T>::Destroy(static_cast<T*>(p));
 }
-#else
+
+template <typename T>
+const uint16_t ArenaContextTraits<T>::id_ =
+    BaseArenaContextTraits::MakeId(DestroyArenaContext<T>);
+
 template <typename T, typename A, typename B>
 struct IfArray {
   using Result = A;
@@ -120,30 +111,40 @@ template <typename T, typename A, typename B>
 struct IfArray<T[], A, B> {
   using Result = B;
 };
-#endif
+
+struct UnrefDestroy {
+  void operator()(const Arena* arena) const;
+};
 
 }  // namespace arena_detail
 
-class Arena {
-#ifndef GRPC_ARENA_POOLED_ALLOCATIONS_USE_MALLOC
-  // Selected pool sizes.
-  // How to tune: see tools/codegen/core/optimize_arena_pool_sizes.py
-  using PoolSizes = absl::integer_sequence<size_t, 80, 304, 528, 1024>;
-  struct FreePoolNode {
-    FreePoolNode* next;
-  };
-#endif
+class ArenaFactory : public RefCounted<ArenaFactory> {
+ public:
+  virtual RefCountedPtr<Arena> MakeArena() = 0;
+  virtual void FinalizeArena(Arena* arena) = 0;
+
+  MemoryAllocator& allocator() { return allocator_; }
+
+ protected:
+  explicit ArenaFactory(MemoryAllocator allocator)
+      : allocator_(std::move(allocator)) {}
+
+ private:
+  MemoryAllocator allocator_;
+};
 
+MemoryAllocator DefaultMemoryAllocatorForSimpleArenaAllocator();
+RefCountedPtr<ArenaFactory> SimpleArenaAllocator(
+    size_t initial_size = 1024,
+    MemoryAllocator allocator =
+        DefaultMemoryAllocatorForSimpleArenaAllocator());
+
+class Arena final : public RefCounted<Arena, NonPolymorphicRefCount,
+                                      arena_detail::UnrefDestroy> {
  public:
   // Create an arena, with \a initial_size bytes in the first allocated buffer.
-  static Arena* Create(size_t initial_size, MemoryAllocator* memory_allocator);
-
-  // Create an arena, with \a initial_size bytes in the first allocated buffer,
-  // and return both a void pointer to the returned arena and a void* with the
-  // first allocation.
-  static std::pair<Arena*, void*> CreateWithAlloc(
-      size_t initial_size, size_t alloc_size,
-      MemoryAllocator* memory_allocator);
+  static RefCountedPtr<Arena> Create(size_t initial_size,
+                                     RefCountedPtr<ArenaFactory> arena_factory);
 
   // Destroy all `ManagedNew` allocated objects.
   // Allows safe destruction of these objects even if they need context held by
@@ -152,9 +153,6 @@ class Arena {
   // TODO(ctiller): eliminate ManagedNew.
   void DestroyManagedNewObjects();
 
-  // Destroy an arena.
-  void Destroy();
-
   // Return the total amount of memory allocated by this arena.
   size_t TotalUsedBytes() const {
     return total_used_.load(std::memory_order_relaxed);
@@ -162,12 +160,10 @@ class Arena {
 
   // Allocate \a size bytes from the arena.
   void* Alloc(size_t size) {
-    static constexpr size_t base_size =
-        GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(Arena));
     size = GPR_ROUND_UP_TO_ALIGNMENT_SIZE(size);
     size_t begin = total_used_.fetch_add(size, std::memory_order_relaxed);
     if (begin + size <= initial_zone_size_) {
-      return reinterpret_cast<char*>(this) + base_size + begin;
+      return reinterpret_cast<char*>(this) + begin;
     } else {
       return AllocZone(size);
     }
@@ -195,95 +191,6 @@ class Arena {
     return &p->t;
   }
 
-#ifndef GRPC_ARENA_POOLED_ALLOCATIONS_USE_MALLOC
-  class PooledDeleter {
-   public:
-    explicit PooledDeleter(std::atomic<FreePoolNode*>* free_list)
-        : free_list_(free_list) {}
-    PooledDeleter() = default;
-    template <typename T>
-    void operator()(T* p) {
-      // TODO(ctiller): promise based filter hijacks ownership of some pointers
-      // to make them appear as PoolPtr without really transferring ownership,
-      // by setting the arena to nullptr.
-      // This is a transitional hack and should be removed once promise based
-      // filter is removed.
-      if (free_list_ != nullptr) {
-        p->~T();
-        FreePooled(p, free_list_);
-      }
-    }
-
-    bool has_freelist() const { return free_list_ != nullptr; }
-
-   private:
-    std::atomic<FreePoolNode*>* free_list_;
-  };
-
-  template <typename T>
-  using PoolPtr = std::unique_ptr<T, PooledDeleter>;
-
-  // Make a unique_ptr to T that is allocated from the arena.
-  // When the pointer is released, the memory may be reused for other
-  // MakePooled(.*) calls.
-  // CAUTION: The amount of memory allocated is rounded up to the nearest
-  //          value in Arena::PoolSizes, and so this may pessimize total
-  //          arena size.
-  template <typename T, typename... Args>
-  PoolPtr<T> MakePooled(Args&&... args) {
-    auto* free_list =
-        &pools_[arena_detail::PoolFromObjectSize<sizeof(T)>(PoolSizes())];
-    return PoolPtr<T>(
-        new (AllocPooled(
-            sizeof(T),
-            arena_detail::AllocationSizeFromObjectSize<sizeof(T)>(PoolSizes()),
-            free_list)) T(std::forward<Args>(args)...),
-        PooledDeleter(free_list));
-  }
-
-  // Make a unique_ptr to an array of T that is allocated from the arena.
-  // When the pointer is released, the memory may be reused for other
-  // MakePooled(.*) calls.
-  // One can use MakePooledArray<char> to allocate a buffer of bytes.
-  // CAUTION: The amount of memory allocated is rounded up to the nearest
-  //          value in Arena::PoolSizes, and so this may pessimize total
-  //          arena size.
-  template <typename T>
-  PoolPtr<T[]> MakePooledArray(size_t n) {
-    auto where =
-        arena_detail::ChoosePoolForAllocationSize(n * sizeof(T), PoolSizes());
-    if (where.pool_index == std::numeric_limits<size_t>::max()) {
-      return PoolPtr<T[]>(new (Alloc(where.alloc_size)) T[n],
-                          PooledDeleter(nullptr));
-    } else {
-      return PoolPtr<T[]>(new (AllocPooled(where.alloc_size, where.alloc_size,
-                                           &pools_[where.pool_index])) T[n],
-                          PooledDeleter(&pools_[where.pool_index]));
-    }
-  }
-
-  // Like MakePooled, but with manual memory management.
-  // The caller is responsible for calling DeletePooled() on the returned
-  // pointer, and expected to call it with the same type T as was passed to this
-  // function (else the free list returned to the arena will be corrupted).
-  template <typename T, typename... Args>
-  T* NewPooled(Args&&... args) {
-    auto* free_list =
-        &pools_[arena_detail::PoolFromObjectSize<sizeof(T)>(PoolSizes())];
-    return new (AllocPooled(
-        sizeof(T),
-        arena_detail::AllocationSizeFromObjectSize<sizeof(T)>(PoolSizes()),
-        free_list)) T(std::forward<Args>(args)...);
-  }
-
-  template <typename T>
-  void DeletePooled(T* p) {
-    auto* free_list =
-        &pools_[arena_detail::PoolFromObjectSize<sizeof(T)>(PoolSizes())];
-    p->~T();
-    FreePooled(p, free_list);
-  }
-#else
   class PooledDeleter {
    public:
     PooledDeleter() = default;
@@ -340,6 +247,11 @@ class Arena {
     return PoolPtr<T>(new T(std::forward<Args>(args)...), PooledDeleter());
   }
 
+  template <typename T>
+  static PoolPtr<T> MakePooledForOverwrite() {
+    return PoolPtr<T>(new T, PooledDeleter());
+  }
+
   // Make a unique_ptr to an array of T that is allocated from the arena.
   // When the pointer is released, the memory may be reused for other
   // MakePooled(.*) calls.
@@ -365,9 +277,37 @@ class Arena {
   void DeletePooled(T* p) {
     delete p;
   }
-#endif
+
+  // Context accessors
+  // Prefer to use the free-standing `GetContext<>` and `SetContext<>` functions
+  // for modern promise-based code -- however legacy filter stack based code
+  // often needs to access these directly.
+  template <typename T>
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION T* GetContext() {
+    return static_cast<T*>(
+        contexts()[arena_detail::ArenaContextTraits<T>::id()]);
+  }
+
+  template <typename T>
+  void SetContext(T* context) {
+    void*& slot = contexts()[arena_detail::ArenaContextTraits<T>::id()];
+    if (slot != nullptr) {
+      ArenaContextType<T>::Destroy(static_cast<T*>(slot));
+    }
+    slot = context;
+    DCHECK_EQ(GetContext<T>(), context);
+  }
+
+  static size_t ArenaOverhead() {
+    return GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(Arena));
+  }
+  static size_t ArenaZoneOverhead() {
+    return GPR_ROUND_UP_TO_ALIGNMENT_SIZE(sizeof(Zone));
+  }
 
  private:
+  friend struct arena_detail::UnrefDestroy;
+
   struct Zone {
     Zone* prev;
   };
@@ -397,41 +337,22 @@ class Arena {
   //   quick optimization (avoiding an atomic fetch-add) for the common case
   //   where we wish to create an arena and then perform an immediate
   //   allocation.
-  explicit Arena(size_t initial_size, size_t initial_alloc,
-                 MemoryAllocator* memory_allocator)
-      : total_used_(GPR_ROUND_UP_TO_ALIGNMENT_SIZE(initial_alloc)),
-        initial_zone_size_(initial_size),
-        memory_allocator_(memory_allocator) {}
+  explicit Arena(size_t initial_size,
+                 RefCountedPtr<ArenaFactory> arena_factory);
 
   ~Arena();
 
   void* AllocZone(size_t size);
-
-#ifndef GRPC_ARENA_POOLED_ALLOCATIONS_USE_MALLOC
-  void* AllocPooled(size_t obj_size, size_t alloc_size,
-                    std::atomic<FreePoolNode*>* head);
-  static void FreePooled(void* p, std::atomic<FreePoolNode*>* head);
-#endif
-
-  void TracePoolAlloc(size_t size, void* ptr) {
-    (void)size;
-    (void)ptr;
-#ifdef GRPC_ARENA_TRACE_POOLED_ALLOCATIONS
-    gpr_log(GPR_ERROR, "ARENA %p ALLOC %" PRIdPTR " @ %p", this, size, ptr);
-#endif
-  }
-  static void TracePoolFree(void* ptr) {
-    (void)ptr;
-#ifdef GRPC_ARENA_TRACE_POOLED_ALLOCATIONS
-    gpr_log(GPR_ERROR, "FREE %p", ptr);
-#endif
+  void Destroy() const;
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION void** contexts() {
+    return reinterpret_cast<void**>(this + 1);
   }
 
   // Keep track of the total used size. We use this in our call sizing
   // hysteresis.
-  std::atomic<size_t> total_used_{0};
-  std::atomic<size_t> total_allocated_{0};
   const size_t initial_zone_size_;
+  std::atomic<size_t> total_used_;
+  std::atomic<size_t> total_allocated_{initial_zone_size_};
   // If the initial arena allocation wasn't enough, we allocate additional zones
   // in a reverse linked list. Each additional zone consists of (1) a pointer to
   // the zone added before this zone (null if this is the first additional zone)
@@ -439,27 +360,34 @@ class Arena {
   // last zone; the zone list is reverse-walked during arena destruction only.
   std::atomic<Zone*> last_zone_{nullptr};
   std::atomic<ManagedNewObject*> managed_new_head_{nullptr};
-#ifndef GRPC_ARENA_POOLED_ALLOCATIONS_USE_MALLOC
-  std::atomic<FreePoolNode*> pools_[PoolSizes::size()]{};
-#endif
-  // The backing memory quota
-  MemoryAllocator* const memory_allocator_;
+  RefCountedPtr<ArenaFactory> arena_factory_;
 };
 
-// Smart pointer for arenas when the final size is not required.
-struct ScopedArenaDeleter {
-  void operator()(Arena* arena) { arena->Destroy(); }
-};
-using ScopedArenaPtr = std::unique_ptr<Arena, ScopedArenaDeleter>;
-inline ScopedArenaPtr MakeScopedArena(size_t initial_size,
-                                      MemoryAllocator* memory_allocator) {
-  return ScopedArenaPtr(Arena::Create(initial_size, memory_allocator));
-}
-
 // Arenas form a context for activities
 template <>
 struct ContextType<Arena> {};
 
+namespace arena_detail {
+inline void UnrefDestroy::operator()(const Arena* arena) const {
+  arena->Destroy();
+}
+}  // namespace arena_detail
+
+namespace promise_detail {
+
+template <typename T>
+class Context<T, absl::void_t<decltype(ArenaContextType<T>::Destroy)>> {
+ public:
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static T* get() {
+    return GetContext<Arena>()->GetContext<T>();
+  }
+  GPR_ATTRIBUTE_ALWAYS_INLINE_FUNCTION static void set(T* value) {
+    GetContext<Arena>()->SetContext(value);
+  }
+};
+
+}  // namespace promise_detail
+
 }  // namespace grpc_core
 
 #endif  // GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_ARENA_H

data/src/core/lib/resource_quota/connection_quota.cc

@@ -12,14 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <grpc/support/port_platform.h>
-
 #include "src/core/lib/resource_quota/connection_quota.h"
 
 #include <atomic>
 #include <cstdint>
 
+#include "absl/log/check.h"
+
 #include <grpc/support/log.h>
+#include <grpc/support/port_platform.h>
 
 namespace grpc_core {
 
@@ -27,10 +28,9 @@ ConnectionQuota::ConnectionQuota() = default;
 
 void ConnectionQuota::SetMaxIncomingConnections(int max_incoming_connections) {
   // The maximum can only be configured once.
-  GPR_ASSERT(max_incoming_connections < INT_MAX);
-  GPR_ASSERT(max_incoming_connections_.exchange(max_incoming_connections,
-                                                std::memory_order_release) ==
-             INT_MAX);
+  CHECK_LT(max_incoming_connections, INT_MAX);
+  CHECK(max_incoming_connections_.exchange(
+            max_incoming_connections, std::memory_order_release) == INT_MAX);
 }
 
 // Returns true if the incoming connection is allowed to be accepted on the
@@ -63,9 +63,8 @@ void ConnectionQuota::ReleaseConnections(int num_connections) {
   if (max_incoming_connections_.load(std::memory_order_relaxed) == INT_MAX) {
     return;
   }
-  GPR_ASSERT(active_incoming_connections_.fetch_sub(
-                 num_connections, std::memory_order_acq_rel) >=
-             num_connections);
+  CHECK(active_incoming_connections_.fetch_sub(
+            num_connections, std::memory_order_acq_rel) >= num_connections);
 }
 
 }  // namespace grpc_core

data/src/core/lib/resource_quota/connection_quota.h

@@ -15,13 +15,13 @@
 #ifndef GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_CONNECTION_QUOTA_H
 #define GRPC_SRC_CORE_LIB_RESOURCE_QUOTA_CONNECTION_QUOTA_H
 
-#include <grpc/support/port_platform.h>
-
 #include <cstddef>
 #include <limits>
 
 #include "absl/base/thread_annotations.h"
 
+#include <grpc/support/port_platform.h>
+
 #include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/gprpp/sync.h"