grpc 1.40.0 → 1.43.1

data/third_party/abseil-cpp/absl/container/inlined_vector.h

@@ -72,37 +72,43 @@ class InlinedVector {
 
   using Storage = inlined_vector_internal::Storage<T, N, A>;
 
-  using AllocatorTraits = typename Storage::AllocatorTraits;
-  using RValueReference = typename Storage::RValueReference;
-  using MoveIterator = typename Storage::MoveIterator;
-  using IsMemcpyOk = typename Storage::IsMemcpyOk;
-
-  template <typename Iterator>
+  template <typename TheA>
+  using AllocatorTraits = inlined_vector_internal::AllocatorTraits<TheA>;
+  template <typename TheA>
+  using MoveIterator = inlined_vector_internal::MoveIterator<TheA>;
+  template <typename TheA>
+  using IsMemcpyOk = inlined_vector_internal::IsMemcpyOk<TheA>;
+
+  template <typename TheA, typename Iterator>
   using IteratorValueAdapter =
-      typename Storage::template IteratorValueAdapter<Iterator>;
-  using CopyValueAdapter = typename Storage::CopyValueAdapter;
-  using DefaultValueAdapter = typename Storage::DefaultValueAdapter;
+      inlined_vector_internal::IteratorValueAdapter<TheA, Iterator>;
+  template <typename TheA>
+  using CopyValueAdapter = inlined_vector_internal::CopyValueAdapter<TheA>;
+  template <typename TheA>
+  using DefaultValueAdapter =
+      inlined_vector_internal::DefaultValueAdapter<TheA>;
 
   template <typename Iterator>
   using EnableIfAtLeastForwardIterator = absl::enable_if_t<
-      inlined_vector_internal::IsAtLeastForwardIterator<Iterator>::value>;
+      inlined_vector_internal::IsAtLeastForwardIterator<Iterator>::value, int>;
   template <typename Iterator>
   using DisableIfAtLeastForwardIterator = absl::enable_if_t<
-      !inlined_vector_internal::IsAtLeastForwardIterator<Iterator>::value>;
+      !inlined_vector_internal::IsAtLeastForwardIterator<Iterator>::value, int>;
 
  public:
-  using allocator_type = typename Storage::allocator_type;
-  using value_type = typename Storage::value_type;
-  using pointer = typename Storage::pointer;
-  using const_pointer = typename Storage::const_pointer;
-  using size_type = typename Storage::size_type;
-  using difference_type = typename Storage::difference_type;
-  using reference = typename Storage::reference;
-  using const_reference = typename Storage::const_reference;
-  using iterator = typename Storage::iterator;
-  using const_iterator = typename Storage::const_iterator;
-  using reverse_iterator = typename Storage::reverse_iterator;
-  using const_reverse_iterator = typename Storage::const_reverse_iterator;
+  using allocator_type = A;
+  using value_type = inlined_vector_internal::ValueType<A>;
+  using pointer = inlined_vector_internal::Pointer<A>;
+  using const_pointer = inlined_vector_internal::ConstPointer<A>;
+  using size_type = inlined_vector_internal::SizeType<A>;
+  using difference_type = inlined_vector_internal::DifferenceType<A>;
+  using reference = inlined_vector_internal::Reference<A>;
+  using const_reference = inlined_vector_internal::ConstReference<A>;
+  using iterator = inlined_vector_internal::Iterator<A>;
+  using const_iterator = inlined_vector_internal::ConstIterator<A>;
+  using reverse_iterator = inlined_vector_internal::ReverseIterator<A>;
+  using const_reverse_iterator =
+      inlined_vector_internal::ConstReverseIterator<A>;
 
   // ---------------------------------------------------------------------------
   // InlinedVector Constructors and Destructor
@@ -111,28 +117,28 @@ class InlinedVector {
   // Creates an empty inlined vector with a value-initialized allocator.
   InlinedVector() noexcept(noexcept(allocator_type())) : storage_() {}
 
-  // Creates an empty inlined vector with a copy of `alloc`.
-  explicit InlinedVector(const allocator_type& alloc) noexcept
-      : storage_(alloc) {}
+  // Creates an empty inlined vector with a copy of `allocator`.
+  explicit InlinedVector(const allocator_type& allocator) noexcept
+      : storage_(allocator) {}
 
   // Creates an inlined vector with `n` copies of `value_type()`.
   explicit InlinedVector(size_type n,
-                         const allocator_type& alloc = allocator_type())
-      : storage_(alloc) {
-    storage_.Initialize(DefaultValueAdapter(), n);
+                         const allocator_type& allocator = allocator_type())
+      : storage_(allocator) {
+    storage_.Initialize(DefaultValueAdapter<A>(), n);
   }
 
   // Creates an inlined vector with `n` copies of `v`.
   InlinedVector(size_type n, const_reference v,
-                const allocator_type& alloc = allocator_type())
-      : storage_(alloc) {
-    storage_.Initialize(CopyValueAdapter(v), n);
+                const allocator_type& allocator = allocator_type())
+      : storage_(allocator) {
+    storage_.Initialize(CopyValueAdapter<A>(std::addressof(v)), n);
   }
 
   // Creates an inlined vector with copies of the elements of `list`.
   InlinedVector(std::initializer_list<value_type> list,
-                const allocator_type& alloc = allocator_type())
-      : InlinedVector(list.begin(), list.end(), alloc) {}
+                const allocator_type& allocator = allocator_type())
+      : InlinedVector(list.begin(), list.end(), allocator) {}
 
   // Creates an inlined vector with elements constructed from the provided
   // forward iterator range [`first`, `last`).
@@ -141,35 +147,36 @@ class InlinedVector {
   // this constructor with two integral arguments and a call to the above
   // `InlinedVector(size_type, const_reference)` constructor.
   template <typename ForwardIterator,
-            EnableIfAtLeastForwardIterator<ForwardIterator>* = nullptr>
+            EnableIfAtLeastForwardIterator<ForwardIterator> = 0>
   InlinedVector(ForwardIterator first, ForwardIterator last,
-                const allocator_type& alloc = allocator_type())
-      : storage_(alloc) {
-    storage_.Initialize(IteratorValueAdapter<ForwardIterator>(first),
+                const allocator_type& allocator = allocator_type())
+      : storage_(allocator) {
+    storage_.Initialize(IteratorValueAdapter<A, ForwardIterator>(first),
                         std::distance(first, last));
   }
 
   // Creates an inlined vector with elements constructed from the provided input
   // iterator range [`first`, `last`).
   template <typename InputIterator,
-            DisableIfAtLeastForwardIterator<InputIterator>* = nullptr>
+            DisableIfAtLeastForwardIterator<InputIterator> = 0>
   InlinedVector(InputIterator first, InputIterator last,
-                const allocator_type& alloc = allocator_type())
-      : storage_(alloc) {
+                const allocator_type& allocator = allocator_type())
+      : storage_(allocator) {
     std::copy(first, last, std::back_inserter(*this));
   }
 
   // Creates an inlined vector by copying the contents of `other` using
   // `other`'s allocator.
   InlinedVector(const InlinedVector& other)
-      : InlinedVector(other, *other.storage_.GetAllocPtr()) {}
+      : InlinedVector(other, other.storage_.GetAllocator()) {}
 
-  // Creates an inlined vector by copying the contents of `other` using `alloc`.
-  InlinedVector(const InlinedVector& other, const allocator_type& alloc)
-      : storage_(alloc) {
+  // Creates an inlined vector by copying the contents of `other` using the
+  // provided `allocator`.
+  InlinedVector(const InlinedVector& other, const allocator_type& allocator)
+      : storage_(allocator) {
     if (other.empty()) {
       // Empty; nothing to do.
-    } else if (IsMemcpyOk::value && !other.storage_.GetIsAllocated()) {
+    } else if (IsMemcpyOk<A>::value && !other.storage_.GetIsAllocated()) {
       // Memcpy-able and do not need allocation.
       storage_.MemcpyFrom(other.storage_);
     } else {
@@ -194,23 +201,23 @@ class InlinedVector {
   InlinedVector(InlinedVector&& other) noexcept(
       absl::allocator_is_nothrow<allocator_type>::value ||
       std::is_nothrow_move_constructible<value_type>::value)
-      : storage_(*other.storage_.GetAllocPtr()) {
-    if (IsMemcpyOk::value) {
+      : storage_(other.storage_.GetAllocator()) {
+    if (IsMemcpyOk<A>::value) {
       storage_.MemcpyFrom(other.storage_);
 
       other.storage_.SetInlinedSize(0);
     } else if (other.storage_.GetIsAllocated()) {
-      storage_.SetAllocatedData(other.storage_.GetAllocatedData(),
-                                other.storage_.GetAllocatedCapacity());
+      storage_.SetAllocation({other.storage_.GetAllocatedData(),
+                              other.storage_.GetAllocatedCapacity()});
       storage_.SetAllocatedSize(other.storage_.GetSize());
 
       other.storage_.SetInlinedSize(0);
     } else {
-      IteratorValueAdapter<MoveIterator> other_values(
-          MoveIterator(other.storage_.GetInlinedData()));
+      IteratorValueAdapter<A, MoveIterator<A>> other_values(
+          MoveIterator<A>(other.storage_.GetInlinedData()));
 
-      inlined_vector_internal::ConstructElements(
-          storage_.GetAllocPtr(), storage_.GetInlinedData(), &other_values,
+      inlined_vector_internal::ConstructElements<A>(
+          storage_.GetAllocator(), storage_.GetInlinedData(), other_values,
           other.storage_.GetSize());
 
       storage_.SetInlinedSize(other.storage_.GetSize());
@@ -218,30 +225,32 @@ class InlinedVector {
   }
 
   // Creates an inlined vector by moving in the contents of `other` with a copy
-  // of `alloc`.
+  // of `allocator`.
   //
-  // NOTE: if `other`'s allocator is not equal to `alloc`, even if `other`
+  // NOTE: if `other`'s allocator is not equal to `allocator`, even if `other`
   // contains allocated memory, this move constructor will still allocate. Since
   // allocation is performed, this constructor can only be `noexcept` if the
   // specified allocator is also `noexcept`.
-  InlinedVector(InlinedVector&& other, const allocator_type& alloc) noexcept(
-      absl::allocator_is_nothrow<allocator_type>::value)
-      : storage_(alloc) {
-    if (IsMemcpyOk::value) {
+  InlinedVector(
+      InlinedVector&& other,
+      const allocator_type& allocator)
+      noexcept(absl::allocator_is_nothrow<allocator_type>::value)
+      : storage_(allocator) {
+    if (IsMemcpyOk<A>::value) {
       storage_.MemcpyFrom(other.storage_);
 
       other.storage_.SetInlinedSize(0);
-    } else if ((*storage_.GetAllocPtr() == *other.storage_.GetAllocPtr()) &&
+    } else if ((storage_.GetAllocator() == other.storage_.GetAllocator()) &&
                other.storage_.GetIsAllocated()) {
-      storage_.SetAllocatedData(other.storage_.GetAllocatedData(),
-                                other.storage_.GetAllocatedCapacity());
+      storage_.SetAllocation({other.storage_.GetAllocatedData(),
+                              other.storage_.GetAllocatedCapacity()});
       storage_.SetAllocatedSize(other.storage_.GetSize());
 
       other.storage_.SetInlinedSize(0);
     } else {
-      storage_.Initialize(
-          IteratorValueAdapter<MoveIterator>(MoveIterator(other.data())),
-          other.size());
+      storage_.Initialize(IteratorValueAdapter<A, MoveIterator<A>>(
+                              MoveIterator<A>(other.data())),
+                          other.size());
     }
   }
 
@@ -442,7 +451,7 @@ class InlinedVector {
   // `InlinedVector::get_allocator()`
   //
   // Returns a copy of the inlined vector's allocator.
-  allocator_type get_allocator() const { return *storage_.GetAllocPtr(); }
+  allocator_type get_allocator() const { return storage_.GetAllocator(); }
 
   // ---------------------------------------------------------------------------
   // InlinedVector Member Mutators
@@ -476,16 +485,16 @@ class InlinedVector {
   // unspecified state.
   InlinedVector& operator=(InlinedVector&& other) {
     if (ABSL_PREDICT_TRUE(this != std::addressof(other))) {
-      if (IsMemcpyOk::value || other.storage_.GetIsAllocated()) {
-        inlined_vector_internal::DestroyElements(storage_.GetAllocPtr(), data(),
-                                                 size());
+      if (IsMemcpyOk<A>::value || other.storage_.GetIsAllocated()) {
+        inlined_vector_internal::DestroyElements<A>(storage_.GetAllocator(),
+                                                    data(), size());
         storage_.DeallocateIfAllocated();
         storage_.MemcpyFrom(other.storage_);
 
         other.storage_.SetInlinedSize(0);
       } else {
-        storage_.Assign(IteratorValueAdapter<MoveIterator>(
-                            MoveIterator(other.storage_.GetInlinedData())),
+        storage_.Assign(IteratorValueAdapter<A, MoveIterator<A>>(
+                            MoveIterator<A>(other.storage_.GetInlinedData())),
                         other.size());
       }
     }
@@ -497,7 +506,7 @@ class InlinedVector {
   //
   // Replaces the contents of the inlined vector with `n` copies of `v`.
   void assign(size_type n, const_reference v) {
-    storage_.Assign(CopyValueAdapter(v), n);
+    storage_.Assign(CopyValueAdapter<A>(std::addressof(v)), n);
   }
 
   // Overload of `InlinedVector::assign(...)` that replaces the contents of the
@@ -511,9 +520,9 @@ class InlinedVector {
   //
   // NOTE: this overload is for iterators that are "forward" category or better.
   template <typename ForwardIterator,
-            EnableIfAtLeastForwardIterator<ForwardIterator>* = nullptr>
+            EnableIfAtLeastForwardIterator<ForwardIterator> = 0>
   void assign(ForwardIterator first, ForwardIterator last) {
-    storage_.Assign(IteratorValueAdapter<ForwardIterator>(first),
+    storage_.Assign(IteratorValueAdapter<A, ForwardIterator>(first),
                     std::distance(first, last));
   }
 
@@ -522,7 +531,7 @@ class InlinedVector {
   //
   // NOTE: this overload is for iterators that are "input" category.
   template <typename InputIterator,
-            DisableIfAtLeastForwardIterator<InputIterator>* = nullptr>
+            DisableIfAtLeastForwardIterator<InputIterator> = 0>
   void assign(InputIterator first, InputIterator last) {
     size_type i = 0;
     for (; i < size() && first != last; ++i, static_cast<void>(++first)) {
@@ -541,7 +550,7 @@ class InlinedVector {
   // is larger than `size()`, new elements are value-initialized.
   void resize(size_type n) {
     ABSL_HARDENING_ASSERT(n <= max_size());
-    storage_.Resize(DefaultValueAdapter(), n);
+    storage_.Resize(DefaultValueAdapter<A>(), n);
   }
 
   // Overload of `InlinedVector::resize(...)` that resizes the inlined vector to
@@ -551,7 +560,7 @@ class InlinedVector {
   // is larger than `size()`, new elements are copied-constructed from `v`.
   void resize(size_type n, const_reference v) {
     ABSL_HARDENING_ASSERT(n <= max_size());
-    storage_.Resize(CopyValueAdapter(v), n);
+    storage_.Resize(CopyValueAdapter<A>(std::addressof(v)), n);
   }
 
   // `InlinedVector::insert(...)`
@@ -564,7 +573,7 @@ class InlinedVector {
 
   // Overload of `InlinedVector::insert(...)` that inserts `v` at `pos` using
   // move semantics, returning an `iterator` to the newly inserted element.
-  iterator insert(const_iterator pos, RValueReference v) {
+  iterator insert(const_iterator pos, value_type&& v) {
     return emplace(pos, std::move(v));
   }
 
@@ -577,7 +586,8 @@ class InlinedVector {
 
     if (ABSL_PREDICT_TRUE(n != 0)) {
       value_type dealias = v;
-      return storage_.Insert(pos, CopyValueAdapter(dealias), n);
+      return storage_.Insert(pos, CopyValueAdapter<A>(std::addressof(dealias)),
+                             n);
     } else {
       return const_cast<iterator>(pos);
     }
@@ -596,14 +606,15 @@ class InlinedVector {
   //
   // NOTE: this overload is for iterators that are "forward" category or better.
   template <typename ForwardIterator,
-            EnableIfAtLeastForwardIterator<ForwardIterator>* = nullptr>
+            EnableIfAtLeastForwardIterator<ForwardIterator> = 0>
   iterator insert(const_iterator pos, ForwardIterator first,
                   ForwardIterator last) {
     ABSL_HARDENING_ASSERT(pos >= begin());
     ABSL_HARDENING_ASSERT(pos <= end());
 
     if (ABSL_PREDICT_TRUE(first != last)) {
-      return storage_.Insert(pos, IteratorValueAdapter<ForwardIterator>(first),
+      return storage_.Insert(pos,
+                             IteratorValueAdapter<A, ForwardIterator>(first),
                              std::distance(first, last));
     } else {
       return const_cast<iterator>(pos);
@@ -616,7 +627,7 @@ class InlinedVector {
   //
   // NOTE: this overload is for iterators that are "input" category.
   template <typename InputIterator,
-            DisableIfAtLeastForwardIterator<InputIterator>* = nullptr>
+            DisableIfAtLeastForwardIterator<InputIterator> = 0>
   iterator insert(const_iterator pos, InputIterator first, InputIterator last) {
     ABSL_HARDENING_ASSERT(pos >= begin());
     ABSL_HARDENING_ASSERT(pos <= end());
@@ -640,8 +651,8 @@ class InlinedVector {
 
     value_type dealias(std::forward<Args>(args)...);
     return storage_.Insert(pos,
-                           IteratorValueAdapter<MoveIterator>(
-                               MoveIterator(std::addressof(dealias))),
+                           IteratorValueAdapter<A, MoveIterator<A>>(
+                               MoveIterator<A>(std::addressof(dealias))),
                            1);
   }
 
@@ -661,7 +672,7 @@ class InlinedVector {
 
   // Overload of `InlinedVector::push_back(...)` for inserting `v` at `end()`
   // using move semantics.
-  void push_back(RValueReference v) {
+  void push_back(value_type&& v) {
     static_cast<void>(emplace_back(std::move(v)));
   }
 
@@ -671,7 +682,7 @@ class InlinedVector {
   void pop_back() noexcept {
     ABSL_HARDENING_ASSERT(!empty());
 
-    AllocatorTraits::destroy(*storage_.GetAllocPtr(), data() + (size() - 1));
+    AllocatorTraits<A>::destroy(storage_.GetAllocator(), data() + (size() - 1));
     storage_.SubtractSize(1);
   }
 
@@ -710,8 +721,8 @@ class InlinedVector {
   // Destroys all elements in the inlined vector, setting the size to `0` and
   // deallocating any held memory.
   void clear() noexcept {
-    inlined_vector_internal::DestroyElements(storage_.GetAllocPtr(), data(),
-                                             size());
+    inlined_vector_internal::DestroyElements<A>(storage_.GetAllocator(), data(),
+                                                size());
     storage_.DeallocateIfAllocated();
 
     storage_.SetInlinedSize(0);
@@ -724,15 +735,12 @@ class InlinedVector {
 
   // `InlinedVector::shrink_to_fit()`
   //
-  // Reduces memory usage by freeing unused memory. After being called, calls to
-  // `capacity()` will be equal to `max(N, size())`.
-  //
-  // If `size() <= N` and the inlined vector contains allocated memory, the
-  // elements will all be moved to the inlined space and the allocated memory
-  // will be deallocated.
+  // Attempts to reduce memory usage by moving elements to (or keeping elements
+  // in) the smallest available buffer sufficient for containing `size()`
+  // elements.
   //
-  // If `size() > N` and `size() < capacity()`, the elements will be moved to a
-  // smaller allocation.
+  // If `size()` is sufficiently small, the elements will be moved into (or kept
+  // in) the inlined space.
   void shrink_to_fit() {
     if (storage_.GetIsAllocated()) {
       storage_.ShrinkToFit();

data/third_party/abseil-cpp/absl/container/internal/hash_function_defaults.h

@@ -78,24 +78,26 @@ struct StringHash {
   }
 };
 
+struct StringEq {
+  using is_transparent = void;
+  bool operator()(absl::string_view lhs, absl::string_view rhs) const {
+    return lhs == rhs;
+  }
+  bool operator()(const absl::Cord& lhs, const absl::Cord& rhs) const {
+    return lhs == rhs;
+  }
+  bool operator()(const absl::Cord& lhs, absl::string_view rhs) const {
+    return lhs == rhs;
+  }
+  bool operator()(absl::string_view lhs, const absl::Cord& rhs) const {
+    return lhs == rhs;
+  }
+};
+
 // Supports heterogeneous lookup for string-like elements.
 struct StringHashEq {
   using Hash = StringHash;
-  struct Eq {
-    using is_transparent = void;
-    bool operator()(absl::string_view lhs, absl::string_view rhs) const {
-      return lhs == rhs;
-    }
-    bool operator()(const absl::Cord& lhs, const absl::Cord& rhs) const {
-      return lhs == rhs;
-    }
-    bool operator()(const absl::Cord& lhs, absl::string_view rhs) const {
-      return lhs == rhs;
-    }
-    bool operator()(absl::string_view lhs, const absl::Cord& rhs) const {
-      return lhs == rhs;
-    }
-  };
+  using Eq = StringEq;
 };
 
 template <>

data/third_party/abseil-cpp/absl/container/internal/hashtablez_sampler.cc

@@ -21,10 +21,11 @@
 #include <limits>
 
 #include "absl/base/attributes.h"
-#include "absl/base/internal/exponential_biased.h"
 #include "absl/container/internal/have_sse.h"
 #include "absl/debugging/stacktrace.h"
 #include "absl/memory/memory.h"
+#include "absl/profiling/internal/exponential_biased.h"
+#include "absl/profiling/internal/sample_recorder.h"
 #include "absl/synchronization/mutex.h"
 
 namespace absl {
@@ -37,10 +38,9 @@ ABSL_CONST_INIT std::atomic<bool> g_hashtablez_enabled{
     false
 };
 ABSL_CONST_INIT std::atomic<int32_t> g_hashtablez_sample_parameter{1 << 10};
-ABSL_CONST_INIT std::atomic<int32_t> g_hashtablez_max_samples{1 << 20};
 
 #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE)
-ABSL_PER_THREAD_TLS_KEYWORD absl::base_internal::ExponentialBiased
+ABSL_PER_THREAD_TLS_KEYWORD absl::profiling_internal::ExponentialBiased
     g_exponential_biased_generator;
 #endif
 
@@ -50,16 +50,14 @@ ABSL_PER_THREAD_TLS_KEYWORD absl::base_internal::ExponentialBiased
 ABSL_PER_THREAD_TLS_KEYWORD int64_t global_next_sample = 0;
 #endif  // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE)
 
-HashtablezSampler& HashtablezSampler::Global() {
+HashtablezSampler& GlobalHashtablezSampler() {
   static auto* sampler = new HashtablezSampler();
   return *sampler;
 }
 
-HashtablezSampler::DisposeCallback HashtablezSampler::SetDisposeCallback(
-    DisposeCallback f) {
-  return dispose_.exchange(f, std::memory_order_relaxed);
-}
-
+// TODO(bradleybear): The comments at this constructors declaration say that the
+// fields are not initialized, but this definition does initialize the fields.
+// Something needs to be cleaned up.
 HashtablezInfo::HashtablezInfo() { PrepareForSampling(); }
 HashtablezInfo::~HashtablezInfo() = default;
 
@@ -73,6 +71,7 @@ void HashtablezInfo::PrepareForSampling() {
   hashes_bitwise_or.store(0, std::memory_order_relaxed);
   hashes_bitwise_and.store(~size_t{}, std::memory_order_relaxed);
   hashes_bitwise_xor.store(0, std::memory_order_relaxed);
+  max_reserve.store(0, std::memory_order_relaxed);
 
   create_time = absl::Now();
   // The inliner makes hardcoded skip_count difficult (especially when combined
@@ -80,93 +79,6 @@ void HashtablezInfo::PrepareForSampling() {
   // instead.
   depth = absl::GetStackTrace(stack, HashtablezInfo::kMaxStackDepth,
                               /* skip_count= */ 0);
-  dead = nullptr;
-}
-
-HashtablezSampler::HashtablezSampler()
-    : dropped_samples_(0), size_estimate_(0), all_(nullptr), dispose_(nullptr) {
-  absl::MutexLock l(&graveyard_.init_mu);
-  graveyard_.dead = &graveyard_;
-}
-
-HashtablezSampler::~HashtablezSampler() {
-  HashtablezInfo* s = all_.load(std::memory_order_acquire);
-  while (s != nullptr) {
-    HashtablezInfo* next = s->next;
-    delete s;
-    s = next;
-  }
-}
-
-void HashtablezSampler::PushNew(HashtablezInfo* sample) {
-  sample->next = all_.load(std::memory_order_relaxed);
-  while (!all_.compare_exchange_weak(sample->next, sample,
-                                     std::memory_order_release,
-                                     std::memory_order_relaxed)) {
-  }
-}
-
-void HashtablezSampler::PushDead(HashtablezInfo* sample) {
-  if (auto* dispose = dispose_.load(std::memory_order_relaxed)) {
-    dispose(*sample);
-  }
-
-  absl::MutexLock graveyard_lock(&graveyard_.init_mu);
-  absl::MutexLock sample_lock(&sample->init_mu);
-  sample->dead = graveyard_.dead;
-  graveyard_.dead = sample;
-}
-
-HashtablezInfo* HashtablezSampler::PopDead() {
-  absl::MutexLock graveyard_lock(&graveyard_.init_mu);
-
-  // The list is circular, so eventually it collapses down to
-  //   graveyard_.dead == &graveyard_
-  // when it is empty.
-  HashtablezInfo* sample = graveyard_.dead;
-  if (sample == &graveyard_) return nullptr;
-
-  absl::MutexLock sample_lock(&sample->init_mu);
-  graveyard_.dead = sample->dead;
-  sample->PrepareForSampling();
-  return sample;
-}
-
-HashtablezInfo* HashtablezSampler::Register() {
-  int64_t size = size_estimate_.fetch_add(1, std::memory_order_relaxed);
-  if (size > g_hashtablez_max_samples.load(std::memory_order_relaxed)) {
-    size_estimate_.fetch_sub(1, std::memory_order_relaxed);
-    dropped_samples_.fetch_add(1, std::memory_order_relaxed);
-    return nullptr;
-  }
-
-  HashtablezInfo* sample = PopDead();
-  if (sample == nullptr) {
-    // Resurrection failed.  Hire a new warlock.
-    sample = new HashtablezInfo();
-    PushNew(sample);
-  }
-
-  return sample;
-}
-
-void HashtablezSampler::Unregister(HashtablezInfo* sample) {
-  PushDead(sample);
-  size_estimate_.fetch_sub(1, std::memory_order_relaxed);
-}
-
-int64_t HashtablezSampler::Iterate(
-    const std::function<void(const HashtablezInfo& stack)>& f) {
-  HashtablezInfo* s = all_.load(std::memory_order_acquire);
-  while (s != nullptr) {
-    absl::MutexLock l(&s->init_mu);
-    if (s->dead == nullptr) {
-      f(*s);
-    }
-    s = s->next;
-  }
-
-  return dropped_samples_.load(std::memory_order_relaxed);
 }
 
 static bool ShouldForceSampling() {
@@ -189,10 +101,12 @@ static bool ShouldForceSampling() {
   return state == kForce;
 }
 
-HashtablezInfo* SampleSlow(int64_t* next_sample) {
+HashtablezInfo* SampleSlow(int64_t* next_sample, size_t inline_element_size) {
   if (ABSL_PREDICT_FALSE(ShouldForceSampling())) {
     *next_sample = 1;
-    return HashtablezSampler::Global().Register();
+    HashtablezInfo* result = GlobalHashtablezSampler().Register();
+    result->inline_element_size = inline_element_size;
+    return result;
   }
 
 #if !defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE)
@@ -214,15 +128,17 @@ HashtablezInfo* SampleSlow(int64_t* next_sample) {
   // that case.
   if (first) {
     if (ABSL_PREDICT_TRUE(--*next_sample > 0)) return nullptr;
-    return SampleSlow(next_sample);
+    return SampleSlow(next_sample, inline_element_size);
   }
 
-  return HashtablezSampler::Global().Register();
+  HashtablezInfo* result = GlobalHashtablezSampler().Register();
+  result->inline_element_size = inline_element_size;
+  return result;
 #endif
 }
 
 void UnsampleSlow(HashtablezInfo* info) {
-  HashtablezSampler::Global().Unregister(info);
+  GlobalHashtablezSampler().Unregister(info);
 }
 
 void RecordInsertSlow(HashtablezInfo* info, size_t hash,
@@ -262,7 +178,7 @@ void SetHashtablezSampleParameter(int32_t rate) {
 
 void SetHashtablezMaxSamples(int32_t max) {
   if (max > 0) {
-    g_hashtablez_max_samples.store(max, std::memory_order_release);
+    GlobalHashtablezSampler().SetMaxSamples(max);
   } else {
     ABSL_RAW_LOG(ERROR, "Invalid hashtablez max samples: %lld",
                  static_cast<long long>(max));  // NOLINT(runtime/int)