grpc 1.60.0 → 1.62.0

data/third_party/abseil-cpp/absl/base/prefetch.h

@@ -24,17 +24,19 @@
 #ifndef ABSL_BASE_PREFETCH_H_
 #define ABSL_BASE_PREFETCH_H_
 
+#include "absl/base/attributes.h"
 #include "absl/base/config.h"
 
 #if defined(ABSL_INTERNAL_HAVE_SSE)
 #include <xmmintrin.h>
 #endif
 
-#if defined(_MSC_VER) && _MSC_VER >= 1900 && \
-    (defined(_M_X64) || defined(_M_IX86))
+#if defined(_MSC_VER)
 #include <intrin.h>
+#if defined(ABSL_INTERNAL_HAVE_SSE)
 #pragma intrinsic(_mm_prefetch)
 #endif
+#endif
 
 namespace absl {
 ABSL_NAMESPACE_BEGIN
@@ -140,21 +142,24 @@ void PrefetchToLocalCacheForWrite(const void* addr);
 // See __builtin_prefetch:
 // https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html.
 //
-inline void PrefetchToLocalCache(const void* addr) {
+ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCache(
+    const void* addr) {
   __builtin_prefetch(addr, 0, 3);
 }
 
-inline void PrefetchToLocalCacheNta(const void* addr) {
+ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheNta(
+    const void* addr) {
   __builtin_prefetch(addr, 0, 0);
 }
 
-inline void PrefetchToLocalCacheForWrite(const void* addr) {
+ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheForWrite(
+    const void* addr) {
   // [x86] gcc/clang don't generate PREFETCHW for __builtin_prefetch(.., 1)
   // unless -march=broadwell or newer; this is not generally the default, so we
   // manually emit prefetchw. PREFETCHW is recognized as a no-op on older Intel
   // processors and has been present on AMD processors since the K6-2.
-#if defined(__x86_64__)
-  asm("prefetchw (%0)" : : "r"(addr));
+#if defined(__x86_64__) && !defined(__PRFCHW__)
+  asm("prefetchw %0" : : "m"(*reinterpret_cast<const char*>(addr)));
 #else
   __builtin_prefetch(addr, 1, 3);
 #endif
@@ -164,15 +169,18 @@ inline void PrefetchToLocalCacheForWrite(const void* addr) {
 
 #define ABSL_HAVE_PREFETCH 1
 
-inline void PrefetchToLocalCache(const void* addr) {
+ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCache(
+    const void* addr) {
   _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_T0);
 }
 
-inline void PrefetchToLocalCacheNta(const void* addr) {
+ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheNta(
+    const void* addr) {
   _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_NTA);
 }
 
-inline void PrefetchToLocalCacheForWrite(const void* addr) {
+ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheForWrite(
+    const void* addr) {
 #if defined(_MM_HINT_ET0)
   _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_ET0);
 #elif !defined(_MSC_VER) && defined(__x86_64__)
@@ -180,15 +188,18 @@ inline void PrefetchToLocalCacheForWrite(const void* addr) {
   // up, PREFETCHW is recognized as a no-op on older Intel processors
   // and has been present on AMD processors since the K6-2. We have this
   // disabled for MSVC compilers as this miscompiles on older MSVC compilers.
-  asm("prefetchw (%0)" : : "r"(addr));
+  asm("prefetchw %0" : : "m"(*reinterpret_cast<const char*>(addr)));
 #endif
 }
 
 #else
 
-inline void PrefetchToLocalCache(const void* addr) {}
-inline void PrefetchToLocalCacheNta(const void* addr) {}
-inline void PrefetchToLocalCacheForWrite(const void* addr) {}
+ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCache(
+    const void* addr) {}
+ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheNta(
+    const void* addr) {}
+ABSL_ATTRIBUTE_ALWAYS_INLINE inline void PrefetchToLocalCacheForWrite(
+    const void* addr) {}
 
 #endif
 

data/third_party/abseil-cpp/absl/base/thread_annotations.h

@@ -36,8 +36,6 @@
 
 #include "absl/base/attributes.h"
 #include "absl/base/config.h"
-// TODO(mbonadei): Remove after the backward compatibility period.
-#include "absl/base/internal/thread_annotations.h"  // IWYU pragma: export
 
 // ABSL_GUARDED_BY()
 //

data/third_party/abseil-cpp/absl/container/flat_hash_map.h

@@ -64,7 +64,7 @@ struct FlatHashMapPolicy;
 //   `insert()`, provided that the map is provided a compatible heterogeneous
 //   hashing function and equality operator.
 // * Invalidates any references and pointers to elements within the table after
-//   `rehash()`.
+//   `rehash()` and when the table is moved.
 // * Contains a `capacity()` member function indicating the number of element
 //   slots (open, deleted, and empty) within the hash map.
 // * Returns `void` from the `erase(iterator)` overload.
@@ -579,9 +579,9 @@ struct FlatHashMapPolicy {
   }
 
   template <class Allocator>
-  static void transfer(Allocator* alloc, slot_type* new_slot,
+  static auto transfer(Allocator* alloc, slot_type* new_slot,
                        slot_type* old_slot) {
-    slot_policy::transfer(alloc, new_slot, old_slot);
+    return slot_policy::transfer(alloc, new_slot, old_slot);
   }
 
   template <class F, class... Args>

data/third_party/abseil-cpp/absl/container/flat_hash_set.h

@@ -60,7 +60,7 @@ struct FlatHashSetPolicy;
 //   that the set is provided a compatible heterogeneous hashing function and
 //   equality operator.
 // * Invalidates any references and pointers to elements within the table after
-//   `rehash()`.
+//   `rehash()` and when the table is moved.
 // * Contains a `capacity()` member function indicating the number of element
 //   slots (open, deleted, and empty) within the hash set.
 // * Returns `void` from the `erase(iterator)` overload.

data/third_party/abseil-cpp/absl/container/internal/common_policy_traits.h

@@ -93,11 +93,13 @@ struct common_policy_traits {
   struct Rank0 : Rank1 {};
 
   // Use auto -> decltype as an enabler.
+  // P::transfer returns std::true_type if transfer uses memcpy (e.g. in
+  // node_slot_policy).
   template <class Alloc, class P = Policy>
   static auto transfer_impl(Alloc* alloc, slot_type* new_slot,
                             slot_type* old_slot, Rank0)
-      -> decltype((void)P::transfer(alloc, new_slot, old_slot)) {
-    P::transfer(alloc, new_slot, old_slot);
+      -> decltype(P::transfer(alloc, new_slot, old_slot)) {
+    return P::transfer(alloc, new_slot, old_slot);
   }
 #if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606
   // This overload returns true_type for the trait below.

data/third_party/abseil-cpp/absl/container/internal/container_memory.h

@@ -122,10 +122,10 @@ auto TupleRefImpl(T&& t, absl::index_sequence<Is...>)
 // Returns a tuple of references to the elements of the input tuple. T must be a
 // tuple.
 template <class T>
-auto TupleRef(T&& t) -> decltype(
-    TupleRefImpl(std::forward<T>(t),
-                 absl::make_index_sequence<
-                     std::tuple_size<typename std::decay<T>::type>::value>())) {
+auto TupleRef(T&& t) -> decltype(TupleRefImpl(
+    std::forward<T>(t),
+    absl::make_index_sequence<
+        std::tuple_size<typename std::decay<T>::type>::value>())) {
   return TupleRefImpl(
       std::forward<T>(t),
       absl::make_index_sequence<
@@ -156,8 +156,8 @@ void ConstructFromTuple(Alloc* alloc, T* ptr, Tuple&& t) {
 // Constructs T using the args specified in the tuple and calls F with the
 // constructed value.
 template <class T, class Tuple, class F>
-decltype(std::declval<F>()(std::declval<T>())) WithConstructed(
-    Tuple&& t, F&& f) {
+decltype(std::declval<F>()(std::declval<T>())) WithConstructed(Tuple&& t,
+                                                               F&& f) {
   return memory_internal::WithConstructedImpl<T>(
       std::forward<Tuple>(t),
       absl::make_index_sequence<
@@ -423,16 +423,19 @@ struct map_slot_policy {
   }
 
   template <class Allocator>
-  static void transfer(Allocator* alloc, slot_type* new_slot,
+  static auto transfer(Allocator* alloc, slot_type* new_slot,
                        slot_type* old_slot) {
+    auto is_relocatable =
+        typename absl::is_trivially_relocatable<value_type>::type();
+
     emplace(new_slot);
 #if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606
-    if (absl::is_trivially_relocatable<value_type>()) {
+    if (is_relocatable) {
       // TODO(b/247130232,b/251814870): remove casts after fixing warnings.
       std::memcpy(static_cast<void*>(std::launder(&new_slot->value)),
                   static_cast<const void*>(&old_slot->value),
                   sizeof(value_type));
-      return;
+      return is_relocatable;
     }
 #endif
 
@@ -444,6 +447,7 @@ struct map_slot_policy {
                                                    std::move(old_slot->value));
     }
     destroy(alloc, old_slot);
+    return is_relocatable;
   }
 };
 

data/third_party/abseil-cpp/absl/container/internal/hashtablez_sampler.h

@@ -137,18 +137,7 @@ class HashtablezInfoHandle {
     UnsampleSlow(info_);
   }
 
-  HashtablezInfoHandle(const HashtablezInfoHandle&) = delete;
-  HashtablezInfoHandle& operator=(const HashtablezInfoHandle&) = delete;
-
-  HashtablezInfoHandle(HashtablezInfoHandle&& o) noexcept
-      : info_(absl::exchange(o.info_, nullptr)) {}
-  HashtablezInfoHandle& operator=(HashtablezInfoHandle&& o) noexcept {
-    if (ABSL_PREDICT_FALSE(info_ != nullptr)) {
-      UnsampleSlow(info_);
-    }
-    info_ = absl::exchange(o.info_, nullptr);
-    return *this;
-  }
+  inline bool IsSampled() const { return ABSL_PREDICT_FALSE(info_ != nullptr); }
 
   inline void RecordStorageChanged(size_t size, size_t capacity) {
     if (ABSL_PREDICT_TRUE(info_ == nullptr)) return;
@@ -198,6 +187,7 @@ class HashtablezInfoHandle {
   explicit HashtablezInfoHandle(std::nullptr_t) {}
 
   inline void Unregister() {}
+  inline bool IsSampled() const { return false; }
   inline void RecordStorageChanged(size_t /*size*/, size_t /*capacity*/) {}
   inline void RecordRehash(size_t /*total_probe_length*/) {}
   inline void RecordReservation(size_t /*target_capacity*/) {}

data/third_party/abseil-cpp/absl/container/internal/inlined_vector.h

@@ -26,6 +26,7 @@
 #include <utility>
 
 #include "absl/base/attributes.h"
+#include "absl/base/config.h"
 #include "absl/base/macros.h"
 #include "absl/container/internal/compressed_tuple.h"
 #include "absl/memory/memory.h"
@@ -384,7 +385,17 @@ class Storage {
 
   bool GetIsAllocated() const { return GetSizeAndIsAllocated() & 1; }
 
-  Pointer<A> GetAllocatedData() { return data_.allocated.allocated_data; }
+  Pointer<A> GetAllocatedData() {
+    // GCC 12 has a false-positive -Wmaybe-uninitialized warning here.
+#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
+#endif
+    return data_.allocated.allocated_data;
+#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0)
+#pragma GCC diagnostic pop
+#endif
+  }
 
   ConstPointer<A> GetAllocatedData() const {
     return data_.allocated.allocated_data;

data/third_party/abseil-cpp/absl/container/internal/layout.h

@@ -55,7 +55,7 @@
 // `Partial()` comes in handy when the array sizes are embedded into the
 // allocation.
 //
-//   // size_t[1] containing N, size_t[1] containing M, double[N], int[M].
+//   // size_t[0] containing N, size_t[1] containing M, double[N], int[M].
 //   using L = Layout<size_t, size_t, double, int>;
 //
 //   unsigned char* Allocate(size_t n, size_t m) {
@@ -172,6 +172,7 @@
 #include <utility>
 
 #include "absl/base/config.h"
+#include "absl/debugging/internal/demangle.h"
 #include "absl/meta/type_traits.h"
 #include "absl/strings/str_cat.h"
 #include "absl/types/span.h"
@@ -181,14 +182,6 @@
 #include <sanitizer/asan_interface.h>
 #endif
 
-#if defined(__GXX_RTTI)
-#define ABSL_INTERNAL_HAS_CXA_DEMANGLE
-#endif
-
-#ifdef ABSL_INTERNAL_HAS_CXA_DEMANGLE
-#include <cxxabi.h>
-#endif
-
 namespace absl {
 ABSL_NAMESPACE_BEGIN
 namespace container_internal {
@@ -294,19 +287,11 @@ constexpr size_t Max(size_t a, size_t b, Ts... rest) {
 template <class T>
 std::string TypeName() {
   std::string out;
-  int status = 0;
-  char* demangled = nullptr;
-#ifdef ABSL_INTERNAL_HAS_CXA_DEMANGLE
-  demangled = abi::__cxa_demangle(typeid(T).name(), nullptr, nullptr, &status);
-#endif
-  if (status == 0 && demangled != nullptr) {  // Demangling succeeded.
-    absl::StrAppend(&out, "<", demangled, ">");
-    free(demangled);
-  } else {
-#if defined(__GXX_RTTI) || defined(_CPPRTTI)
-    absl::StrAppend(&out, "<", typeid(T).name(), ">");
+#if ABSL_INTERNAL_HAS_RTTI
+  absl::StrAppend(&out, "<",
+                  absl::debugging_internal::DemangleString(typeid(T).name()),
+                  ">");
 #endif
-  }
   return out;
 }
 

data/third_party/abseil-cpp/absl/container/internal/raw_hash_map.h

@@ -19,6 +19,8 @@
 #include <type_traits>
 #include <utility>
 
+#include "absl/base/attributes.h"
+#include "absl/base/config.h"
 #include "absl/base/internal/throw_delegate.h"
 #include "absl/container/internal/container_memory.h"
 #include "absl/container/internal/raw_hash_set.h"  // IWYU pragma: export
@@ -175,13 +177,20 @@ class raw_hash_map : public raw_hash_set<Policy, Hash, Eq, Alloc> {
   template <class K = key_type, class P = Policy, K* = nullptr>
   MappedReference<P> operator[](key_arg<K>&& key)
       ABSL_ATTRIBUTE_LIFETIME_BOUND {
-    return Policy::value(&*try_emplace(std::forward<K>(key)).first);
+    // It is safe to use unchecked_deref here because try_emplace
+    // will always return an iterator pointing to a valid item in the table,
+    // since it inserts if nothing is found for the given key.
+    return Policy::value(
+        &this->unchecked_deref(try_emplace(std::forward<K>(key)).first));
   }
 
   template <class K = key_type, class P = Policy>
   MappedReference<P> operator[](const key_arg<K>& key)
       ABSL_ATTRIBUTE_LIFETIME_BOUND {
-    return Policy::value(&*try_emplace(key).first);
+    // It is safe to use unchecked_deref here because try_emplace
+    // will always return an iterator pointing to a valid item in the table,
+    // since it inserts if nothing is found for the given key.
+    return Policy::value(&this->unchecked_deref(try_emplace(key).first));
   }
 
  private:

data/third_party/abseil-cpp/absl/container/internal/raw_hash_set.cc

@@ -17,11 +17,13 @@
 #include <atomic>
 #include <cassert>
 #include <cstddef>
+#include <cstdint>
 #include <cstring>
 
 #include "absl/base/attributes.h"
 #include "absl/base/config.h"
 #include "absl/base/dynamic_annotations.h"
+#include "absl/container/internal/container_memory.h"
 #include "absl/hash/hash.h"
 
 namespace absl {
@@ -67,6 +69,16 @@ inline size_t RandomSeed() {
   return value ^ static_cast<size_t>(reinterpret_cast<uintptr_t>(&counter));
 }
 
+bool ShouldRehashForBugDetection(const ctrl_t* ctrl, size_t capacity) {
+  // Note: we can't use the abseil-random library because abseil-random
+  // depends on swisstable. We want to return true with probability
+  // `min(1, RehashProbabilityConstant() / capacity())`. In order to do this,
+  // we probe based on a random hash and see if the offset is less than
+  // RehashProbabilityConstant().
+  return probe(ctrl, capacity, absl::HashOf(RandomSeed())).offset() <
+         RehashProbabilityConstant();
+}
+
 }  // namespace
 
 GenerationType* EmptyGeneration() {
@@ -84,13 +96,12 @@ bool CommonFieldsGenerationInfoEnabled::
                                               size_t capacity) const {
   if (reserved_growth_ == kReservedGrowthJustRanOut) return true;
   if (reserved_growth_ > 0) return false;
-  // Note: we can't use the abseil-random library because abseil-random
-  // depends on swisstable. We want to return true with probability
-  // `min(1, RehashProbabilityConstant() / capacity())`. In order to do this,
-  // we probe based on a random hash and see if the offset is less than
-  // RehashProbabilityConstant().
-  return probe(ctrl, capacity, absl::HashOf(RandomSeed())).offset() <
-         RehashProbabilityConstant();
+  return ShouldRehashForBugDetection(ctrl, capacity);
+}
+
+bool CommonFieldsGenerationInfoEnabled::should_rehash_for_bug_detection_on_move(
+    const ctrl_t* ctrl, size_t capacity) const {
+  return ShouldRehashForBugDetection(ctrl, capacity);
 }
 
 bool ShouldInsertBackwards(size_t hash, const ctrl_t* ctrl) {
@@ -117,14 +128,6 @@ FindInfo find_first_non_full_outofline(const CommonFields& common,
   return find_first_non_full(common, hash);
 }
 
-// Returns the address of the ith slot in slots where each slot occupies
-// slot_size.
-static inline void* SlotAddress(void* slot_array, size_t slot,
-                                size_t slot_size) {
-  return reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(slot_array) +
-                                 (slot * slot_size));
-}
-
 // Returns the address of the slot just after slot assuming each slot has the
 // specified size.
 static inline void* NextSlot(void* slot, size_t slot_size) {
@@ -218,26 +221,35 @@ void DropDeletesWithoutResize(CommonFields& common,
   common.infoz().RecordRehash(total_probe_length);
 }
 
-void EraseMetaOnly(CommonFields& c, ctrl_t* it, size_t slot_size) {
-  assert(IsFull(*it) && "erasing a dangling iterator");
-  c.set_size(c.size() - 1);
-  const auto index = static_cast<size_t>(it - c.control());
+static bool WasNeverFull(CommonFields& c, size_t index) {
+  if (is_single_group(c.capacity())) {
+    return true;
+  }
   const size_t index_before = (index - Group::kWidth) & c.capacity();
-  const auto empty_after = Group(it).MaskEmpty();
+  const auto empty_after = Group(c.control() + index).MaskEmpty();
   const auto empty_before = Group(c.control() + index_before).MaskEmpty();
 
   // We count how many consecutive non empties we have to the right and to the
   // left of `it`. If the sum is >= kWidth then there is at least one probe
   // window that might have seen a full group.
-  bool was_never_full = empty_before && empty_after &&
-                        static_cast<size_t>(empty_after.TrailingZeros()) +
-                                empty_before.LeadingZeros() <
-                            Group::kWidth;
-
-  SetCtrl(c, index, was_never_full ? ctrl_t::kEmpty : ctrl_t::kDeleted,
-          slot_size);
-  c.set_growth_left(c.growth_left() + (was_never_full ? 1 : 0));
+  return empty_before && empty_after &&
+         static_cast<size_t>(empty_after.TrailingZeros()) +
+                 empty_before.LeadingZeros() <
+             Group::kWidth;
+}
+
+void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size) {
+  assert(IsFull(c.control()[index]) && "erasing a dangling iterator");
+  c.decrement_size();
   c.infoz().RecordErase();
+
+  if (WasNeverFull(c, index)) {
+    SetCtrl(c, index, ctrl_t::kEmpty, slot_size);
+    c.set_growth_left(c.growth_left() + 1);
+    return;
+  }
+
+  SetCtrl(c, index, ctrl_t::kDeleted, slot_size);
 }
 
 void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy,
@@ -245,19 +257,124 @@ void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy,
   c.set_size(0);
   if (reuse) {
     ResetCtrl(c, policy.slot_size);
+    ResetGrowthLeft(c);
     c.infoz().RecordStorageChanged(0, c.capacity());
   } else {
+    // We need to record infoz before calling dealloc, which will unregister
+    // infoz.
+    c.infoz().RecordClearedReservation();
+    c.infoz().RecordStorageChanged(0, 0);
     (*policy.dealloc)(c, policy);
     c.set_control(EmptyGroup());
     c.set_generation_ptr(EmptyGeneration());
     c.set_slots(nullptr);
     c.set_capacity(0);
-    c.infoz().RecordClearedReservation();
-    assert(c.size() == 0);
-    c.infoz().RecordStorageChanged(0, 0);
   }
 }
 
+void HashSetResizeHelper::GrowIntoSingleGroupShuffleControlBytes(
+    ctrl_t* new_ctrl, size_t new_capacity) const {
+  assert(is_single_group(new_capacity));
+  constexpr size_t kHalfWidth = Group::kWidth / 2;
+  assert(old_capacity_ < kHalfWidth);
+
+  const size_t half_old_capacity = old_capacity_ / 2;
+
+  // NOTE: operations are done with compile time known size = kHalfWidth.
+  // Compiler optimizes that into single ASM operation.
+
+  // Copy second half of bytes to the beginning.
+  // We potentially copy more bytes in order to have compile time known size.
+  // Mirrored bytes from the old_ctrl_ will also be copied.
+  // In case of old_capacity_ == 3, we will copy 1st element twice.
+  // Examples:
+  // old_ctrl = 0S0EEEEEEE...
+  // new_ctrl = S0EEEEEEEE...
+  //
+  // old_ctrl = 01S01EEEEE...
+  // new_ctrl = 1S01EEEEEE...
+  //
+  // old_ctrl = 0123456S0123456EE...
+  // new_ctrl = 456S0123?????????...
+  std::memcpy(new_ctrl, old_ctrl_ + half_old_capacity + 1, kHalfWidth);
+  // Clean up copied kSentinel from old_ctrl.
+  new_ctrl[half_old_capacity] = ctrl_t::kEmpty;
+
+  // Clean up damaged or uninitialized bytes.
+
+  // Clean bytes after the intended size of the copy.
+  // Example:
+  // new_ctrl = 1E01EEEEEEE????
+  // *new_ctrl= 1E0EEEEEEEE????
+  // position      /
+  std::memset(new_ctrl + old_capacity_ + 1, static_cast<int8_t>(ctrl_t::kEmpty),
+              kHalfWidth);
+  // Clean non-mirrored bytes that are not initialized.
+  // For small old_capacity that may be inside of mirrored bytes zone.
+  // Examples:
+  // new_ctrl = 1E0EEEEEEEE??????????....
+  // *new_ctrl= 1E0EEEEEEEEEEEEE?????....
+  // position           /
+  //
+  // new_ctrl = 456E0123???????????...
+  // *new_ctrl= 456E0123EEEEEEEE???...
+  // position           /
+  std::memset(new_ctrl + kHalfWidth, static_cast<int8_t>(ctrl_t::kEmpty),
+              kHalfWidth);
+  // Clean last mirrored bytes that are not initialized
+  // and will not be overwritten by mirroring.
+  // Examples:
+  // new_ctrl = 1E0EEEEEEEEEEEEE????????
+  // *new_ctrl= 1E0EEEEEEEEEEEEEEEEEEEEE
+  // position           S       /
+  //
+  // new_ctrl = 456E0123EEEEEEEE???????????????
+  // *new_ctrl= 456E0123EEEEEEEE???????EEEEEEEE
+  // position                  S       /
+  std::memset(new_ctrl + new_capacity + kHalfWidth,
+              static_cast<int8_t>(ctrl_t::kEmpty), kHalfWidth);
+
+  // Create mirrored bytes. old_capacity_ < kHalfWidth
+  // Example:
+  // new_ctrl = 456E0123EEEEEEEE???????EEEEEEEE
+  // *new_ctrl= 456E0123EEEEEEEE456E0123EEEEEEE
+  // position                  S/
+  ctrl_t g[kHalfWidth];
+  std::memcpy(g, new_ctrl, kHalfWidth);
+  std::memcpy(new_ctrl + new_capacity + 1, g, kHalfWidth);
+
+  // Finally set sentinel to its place.
+  new_ctrl[new_capacity] = ctrl_t::kSentinel;
+}
+
+void HashSetResizeHelper::GrowIntoSingleGroupShuffleTransferableSlots(
+    void* old_slots, void* new_slots, size_t slot_size) const {
+  assert(old_capacity_ > 0);
+  const size_t half_old_capacity = old_capacity_ / 2;
+
+  SanitizerUnpoisonMemoryRegion(old_slots, slot_size * old_capacity_);
+  std::memcpy(new_slots,
+              SlotAddress(old_slots, half_old_capacity + 1, slot_size),
+              slot_size * half_old_capacity);
+  std::memcpy(SlotAddress(new_slots, half_old_capacity + 1, slot_size),
+              old_slots, slot_size * (half_old_capacity + 1));
+}
+
+void HashSetResizeHelper::GrowSizeIntoSingleGroupTransferable(
+    CommonFields& c, void* old_slots, size_t slot_size) {
+  assert(old_capacity_ < Group::kWidth / 2);
+  assert(is_single_group(c.capacity()));
+  assert(IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity()));
+
+  GrowIntoSingleGroupShuffleControlBytes(c.control(), c.capacity());
+  GrowIntoSingleGroupShuffleTransferableSlots(old_slots, c.slot_array(),
+                                              slot_size);
+
+  // We poison since GrowIntoSingleGroupShuffleTransferableSlots
+  // may leave empty slots unpoisoned.
+  PoisonSingleGroupEmptySlots(c, slot_size);
+}
+
 }  // namespace container_internal
 ABSL_NAMESPACE_END
 }  // namespace absl