grpc 1.84.0 → 1.85.0.pre1

data/third_party/abseil-cpp/absl/container/internal/raw_hash_set.cc

@@ -19,6 +19,9 @@
 #include <cstddef>
 #include <cstdint>
 #include <cstring>
+#include <memory>
+#include <tuple>
+#include <utility>
 
 #include "absl/base/attributes.h"
 #include "absl/base/config.h"
@@ -40,32 +43,13 @@ namespace container_internal {
 // Represents a control byte corresponding to a full slot with arbitrary hash.
 constexpr ctrl_t ZeroCtrlT() { return static_cast<ctrl_t>(0); }
 
-// We have space for `growth_info` before a single block of control bytes. A
-// single block of empty control bytes for tables without any slots allocated.
-// This enables removing a branch in the hot path of find(). In order to ensure
-// that the control bytes are aligned to 16, we have 16 bytes before the control
-// bytes even though growth_info only needs 8.
-alignas(16) ABSL_CONST_INIT ABSL_DLL const ctrl_t kEmptyGroup[32] = {
-    ZeroCtrlT(),       ZeroCtrlT(),    ZeroCtrlT(),    ZeroCtrlT(),
-    ZeroCtrlT(),       ZeroCtrlT(),    ZeroCtrlT(),    ZeroCtrlT(),
-    ZeroCtrlT(),       ZeroCtrlT(),    ZeroCtrlT(),    ZeroCtrlT(),
-    ZeroCtrlT(),       ZeroCtrlT(),    ZeroCtrlT(),    ZeroCtrlT(),
-    ctrl_t::kSentinel, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty,
-    ctrl_t::kEmpty,    ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty,
-    ctrl_t::kEmpty,    ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty,
-    ctrl_t::kEmpty,    ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty};
-
-// We need one full byte followed by a sentinel byte for iterator::operator++ to
-// work. We have a full group after kSentinel to be safe (in case operator++ is
-// changed to read a full group).
-ABSL_CONST_INIT ABSL_DLL const ctrl_t kSooControl[17] = {
-    ZeroCtrlT(),    ctrl_t::kSentinel, ZeroCtrlT(),    ctrl_t::kEmpty,
-    ctrl_t::kEmpty, ctrl_t::kEmpty,    ctrl_t::kEmpty, ctrl_t::kEmpty,
-    ctrl_t::kEmpty, ctrl_t::kEmpty,    ctrl_t::kEmpty, ctrl_t::kEmpty,
-    ctrl_t::kEmpty, ctrl_t::kEmpty,    ctrl_t::kEmpty, ctrl_t::kEmpty,
-    ctrl_t::kEmpty};
-static_assert(NumControlBytes(SooCapacity()) <= 17,
-              "kSooControl capacity too small");
+// A single control byte for default-constructed iterators. We leave it
+// uninitialized because reading this memory is a bug.
+ABSL_DLL ctrl_t kDefaultIterControl;
+
+// We need one full byte followed by a sentinel byte for iterator::operator++.
+ABSL_CONST_INIT ABSL_DLL const ctrl_t kSooControl[2] = {ZeroCtrlT(),
+                                                        ctrl_t::kSentinel};
 
 namespace {
 
@@ -78,14 +62,15 @@ namespace {
   assert((CONDITION) && "Try enabling sanitizers.")
 #endif
 
-[[noreturn]] ABSL_ATTRIBUTE_NOINLINE void HashTableSizeOverflow() {
-  ABSL_RAW_LOG(FATAL, "Hash table size overflow");
+void ValidateMaxSize([[maybe_unused]] size_t size,
+                     [[maybe_unused]] size_t key_size,
+                     [[maybe_unused]] size_t slot_size) {
+  ABSL_SWISSTABLE_ASSERT(size <= MaxValidSize(key_size, slot_size));
 }
-
-void ValidateMaxSize(size_t size, size_t slot_size) {
-  if (IsAboveValidSize(size, slot_size)) {
-    HashTableSizeOverflow();
-  }
+void ValidateMaxCapacity(size_t capacity, size_t key_size, size_t slot_size) {
+  if (capacity <= 1) return;
+  ValidateMaxSize(CapacityToGrowth(PreviousCapacity(capacity)), key_size,
+                  slot_size);
 }
 
 // Returns "random" seed.
@@ -100,14 +85,14 @@ inline size_t RandomSeed() {
   return value ^ static_cast<size_t>(reinterpret_cast<uintptr_t>(&counter));
 }
 
-bool ShouldRehashForBugDetection(PerTableSeed seed, size_t capacity) {
+bool ShouldRehashForBugDetection(size_t capacity) {
   // Note: we can't use the abseil-random library because abseil-random
   // depends on swisstable. We want to return true with probability
   // `min(1, RehashProbabilityConstant() / capacity())`. In order to do this,
   // we probe based on a random hash and see if the offset is less than
   // RehashProbabilityConstant().
-  return probe(seed, capacity, absl::HashOf(RandomSeed())).offset() <
-         RehashProbabilityConstant();
+  return probe(HashtableCapacity(capacity), absl::HashOf(RandomSeed()))
+             .offset() < RehashProbabilityConstant();
 }
 
 // Find a non-deterministic hash for single group table.
@@ -118,6 +103,21 @@ size_t SingleGroupTableH1(size_t hash, PerTableSeed seed) {
   return hash ^ seed.seed();
 }
 
+// Returns the offset of the new element after resize from capacity 1 to 3.
+size_t Resize1To3NewOffset(size_t hash, PerTableSeed seed) {
+  // After resize from capacity 1 to 3, we always have exactly the slot with
+  // index 1 occupied, so we need to insert either at index 0 or index 2.
+  static_assert(SooSlotIndex() == 1);
+  return SingleGroupTableH1(hash, seed) & 2;
+}
+
+// Returns the address of the ith slot in slots where each slot occupies
+// slot_size.
+inline void* SlotAddress(void* slot_array, size_t slot, size_t slot_size) {
+  return static_cast<void*>(static_cast<char*>(slot_array) +
+                            (slot * slot_size));
+}
+
 // Returns the address of the slot `i` iterations after `slot` assuming each
 // slot has the specified size.
 inline void* NextSlot(void* slot, size_t slot_size, size_t i = 1) {
@@ -133,6 +133,16 @@ inline void* PrevSlot(void* slot, size_t slot_size) {
 
 }  // namespace
 
+// Must be defined out-of-line to avoid MSVC error C2482 on some platforms,
+// which is caused by non-constexpr initialization.
+uint16_t NextHashTableSeed() {
+  static_assert(PerTableSeed::kBitCount <= 16);
+  thread_local uint16_t seed =
+      static_cast<uint16_t>(reinterpret_cast<uintptr_t>(&seed));
+  seed += uint16_t{0xad53};
+  return seed;
+}
+
 GenerationType* EmptyGeneration() {
   if (SwisstableGenerationsEnabled()) {
     constexpr size_t kNumEmptyGenerations = 1024;
@@ -144,66 +154,98 @@ GenerationType* EmptyGeneration() {
 }
 
 bool CommonFieldsGenerationInfoEnabled::
-    should_rehash_for_bug_detection_on_insert(PerTableSeed seed,
-                                              size_t capacity) const {
+    should_rehash_for_bug_detection_on_insert(size_t capacity) const {
   if (reserved_growth_ == kReservedGrowthJustRanOut) return true;
   if (reserved_growth_ > 0) return false;
-  return ShouldRehashForBugDetection(seed, capacity);
+  return ShouldRehashForBugDetection(capacity);
 }
 
 bool CommonFieldsGenerationInfoEnabled::should_rehash_for_bug_detection_on_move(
-    PerTableSeed seed, size_t capacity) const {
-  return ShouldRehashForBugDetection(seed, capacity);
+    size_t capacity) const {
+  return ShouldRehashForBugDetection(capacity);
 }
 
 namespace {
 
-FindInfo find_first_non_full_from_h1(const ctrl_t* ctrl, size_t h1,
-                                     size_t capacity) {
-  auto seq = probe(h1, capacity);
-  if (IsEmptyOrDeleted(ctrl[seq.offset()])) {
-    return {seq.offset(), /*probe_length=*/0};
-  }
+// Probes an array of control bits using a probe sequence,
+// and returns the mask corresponding to the first group with a deleted or empty
+// slot.
+inline Group::NonIterableBitMaskType probe_till_first_non_full_group(
+    const ctrl_t* ctrl, probe_seq<Group::kWidth>& seq,
+    [[maybe_unused]] size_t capacity) {
   while (true) {
     GroupFullEmptyOrDeleted g{ctrl + seq.offset()};
     auto mask = g.MaskEmptyOrDeleted();
     if (mask) {
-      return {seq.offset(mask.LowestBitSet()), seq.index()};
+      return mask;
     }
     seq.next();
     ABSL_SWISSTABLE_ASSERT(seq.index() <= capacity && "full table!");
   }
 }
 
-// Whether a table is "small". A small table fits entirely into a probing
-// group, i.e., has a capacity < `Group::kWidth`.
+FindInfo find_first_non_full_from_h1(const ctrl_t* ctrl, size_t h1,
+                                     HashtableCapacity capacity) {
+  auto seq = probe_h1(capacity, h1);
+  if (IsEmptyOrDeleted(ctrl[seq.offset()])) {
+    return {seq.offset(), /*probe_length=*/0};
+  }
+  auto mask = probe_till_first_non_full_group(ctrl, seq, capacity.capacity());
+  return {seq.offset(mask.LowestBitSet()), seq.index()};
+}
+
+// Probes an array of control bits using a probe sequence derived from `hash`,
+// and returns the offset corresponding to the first deleted or empty slot.
+//
+// Behavior when the entire table is full is undefined.
+//
+// NOTE: this function must work with tables having both empty and deleted
+// slots in the same group. Such tables appear during `erase()`.
+FindInfo find_first_non_full(const CommonFields& common, size_t hash) {
+  return find_first_non_full_from_h1(common.control(), H1(hash),
+                                     common.capacity_impl());
+}
+
+// Same as `find_first_non_full`, but returns the mask corresponding to the
+// first group with a deleted or empty slot.
+std::pair<FindInfo, Group::NonIterableBitMaskType> find_first_non_full_group(
+    const CommonFields& common, size_t hash) {
+  auto seq = probe(common, hash);
+  auto mask =
+      probe_till_first_non_full_group(common.control(), seq, common.capacity());
+  return {{seq.offset(), seq.index()}, mask};
+}
+
+// Whether a table fits in half a group. A half-group table fits entirely into a
+// probing group, i.e., has a capacity < `Group::kWidth`.
 //
-// In small mode we are able to use the whole capacity. The extra control
+// In half-group mode we are able to use the whole capacity. The extra control
 // bytes give us at least one "empty" control byte to stop the iteration.
 // This is important to make 1 a valid capacity.
 //
-// In small mode only the first `capacity` control bytes after the sentinel
+// In half-group mode only the first `capacity` control bytes after the sentinel
 // are valid. The rest contain dummy ctrl_t::kEmpty values that do not
 // represent a real slot.
-constexpr bool is_small(size_t capacity) {
+constexpr bool is_half_group(size_t capacity) {
   return capacity < Group::kWidth - 1;
 }
 
 template <class Fn>
 void IterateOverFullSlotsImpl(const CommonFields& c, size_t slot_size, Fn cb) {
   const size_t cap = c.capacity();
+  ABSL_SWISSTABLE_ASSERT(!IsSmallCapacity(cap));
   const ctrl_t* ctrl = c.control();
   void* slot = c.slot_array();
-  if (is_small(cap)) {
-    // Mirrored/cloned control bytes in small table are also located in the
+  if (is_half_group(cap)) {
+    // Mirrored/cloned control bytes in half-group table are also located in the
     // first group (starting from position 0). We are taking group from position
     // `capacity` in order to avoid duplicates.
 
-    // Small tables capacity fits into portable group, where
+    // Half-group tables capacity fits into portable group, where
     // GroupPortableImpl::MaskFull is more efficient for the
     // capacity <= GroupPortableImpl::kWidth.
     ABSL_SWISSTABLE_ASSERT(cap <= GroupPortableImpl::kWidth &&
-                           "unexpectedly large small capacity");
+                           "unexpectedly large half-group capacity");
     static_assert(Group::kWidth >= GroupPortableImpl::kWidth,
                   "unexpected group width");
     // Group starts from kSentinel slot, so indices in the mask will
@@ -250,11 +292,6 @@ void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity) {
   ctrl[capacity] = ctrl_t::kSentinel;
 }
 
-FindInfo find_first_non_full(const CommonFields& common, size_t hash) {
-  return find_first_non_full_from_h1(common.control(), H1(hash, common.seed()),
-                                     common.capacity());
-}
-
 void IterateOverFullSlots(const CommonFields& c, size_t slot_size,
                           absl::FunctionRef<void(const ctrl_t*, void*)> cb) {
   IterateOverFullSlotsImpl(c, slot_size, cb);
@@ -300,6 +337,69 @@ void PrepareInsertCommon(CommonFields& common) {
   common.maybe_increment_generation_on_insert();
 }
 
+// Sets sanitizer poisoning for slot corresponding to control byte being set.
+inline void DoSanitizeOnSetCtrl(const CommonFields& c, size_t i, ctrl_t h,
+                                size_t slot_size) {
+  ABSL_SWISSTABLE_ASSERT(i < c.capacity());
+  auto* slot_i = static_cast<const char*>(c.slot_array()) + i * slot_size;
+  if (IsFull(h)) {
+    SanitizerUnpoisonMemoryRegion(slot_i, slot_size);
+  } else {
+    SanitizerPoisonMemoryRegion(slot_i, slot_size);
+  }
+}
+
+// Sets `ctrl[i]` to `h`.
+//
+// Unlike setting it directly, this function will perform bounds checks and
+// mirror the value to the cloned tail if necessary.
+inline void SetCtrl(const CommonFields& c, size_t i, ctrl_t h,
+                    size_t slot_size) {
+  ABSL_SWISSTABLE_ASSERT(!c.is_small());
+  DoSanitizeOnSetCtrl(c, i, h, slot_size);
+  ctrl_t* ctrl = c.control();
+  ctrl[i] = h;
+  ctrl[((i - NumClonedBytes()) & c.capacity()) +
+       (NumClonedBytes() & c.capacity())] = h;
+}
+// Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`.
+inline void SetCtrl(const CommonFields& c, size_t i, h2_t h, size_t slot_size) {
+  SetCtrl(c, i, static_cast<ctrl_t>(h), slot_size);
+}
+
+// Like SetCtrl, but in a single group table, we can save some operations when
+// setting the cloned control byte.
+inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, ctrl_t h,
+                                      size_t slot_size) {
+  ABSL_SWISSTABLE_ASSERT(!c.is_small());
+  ABSL_SWISSTABLE_ASSERT(is_single_group(c.capacity()));
+  DoSanitizeOnSetCtrl(c, i, h, slot_size);
+  ctrl_t* ctrl = c.control();
+  ctrl[i] = h;
+  ctrl[i + c.capacity() + 1] = h;
+}
+// Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`.
+inline void SetCtrlInSingleGroupTable(const CommonFields& c, size_t i, h2_t h,
+                                      size_t slot_size) {
+  SetCtrlInSingleGroupTable(c, i, static_cast<ctrl_t>(h), slot_size);
+}
+
+// Like SetCtrl, but in a table with capacity >= Group::kWidth - 1,
+// we can save some operations when setting the cloned control byte.
+inline void SetCtrlInLargeTable(const CommonFields& c, size_t i, ctrl_t h,
+                                size_t slot_size) {
+  ABSL_SWISSTABLE_ASSERT(c.capacity() >= Group::kWidth - 1);
+  DoSanitizeOnSetCtrl(c, i, h, slot_size);
+  ctrl_t* ctrl = c.control();
+  ctrl[i] = h;
+  ctrl[((i - NumClonedBytes()) & c.capacity()) + NumClonedBytes()] = h;
+}
+// Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`.
+inline void SetCtrlInLargeTable(const CommonFields& c, size_t i, h2_t h,
+                                size_t slot_size) {
+  SetCtrlInLargeTable(c, i, static_cast<ctrl_t>(h), slot_size);
+}
+
 size_t DropDeletesWithoutResizeAndPrepareInsert(
     CommonFields& common, const PolicyFunctions& __restrict policy,
     size_t new_hash) {
@@ -347,7 +447,7 @@ size_t DropDeletesWithoutResizeAndPrepareInsert(
       continue;
     }
     if (!IsDeleted(ctrl[i])) continue;
-    const size_t hash = (*hasher)(hash_fn, slot_ptr);
+    const size_t hash = (*hasher)(hash_fn, slot_ptr, common.seed().seed());
     const FindInfo target = find_first_non_full(common, hash);
     const size_t new_i = target.offset;
     total_probe_length += target.probe_length;
@@ -405,12 +505,12 @@ size_t DropDeletesWithoutResizeAndPrepareInsert(
   ResetGrowthLeft(common);
   FindInfo find_info = find_first_non_full(common, new_hash);
   SetCtrlInLargeTable(common, find_info.offset, H2(new_hash), slot_size);
-  common.infoz().RecordInsert(new_hash, find_info.probe_length);
+  common.infoz().RecordInsertMiss(new_hash, find_info.probe_length);
   common.infoz().RecordRehash(total_probe_length);
   return find_info.offset;
 }
 
-static bool WasNeverFull(CommonFields& c, size_t index) {
+bool WasNeverFull(CommonFields& c, size_t index) {
   if (is_single_group(c.capacity())) {
     return true;
   }
@@ -434,6 +534,7 @@ void ResetCtrl(CommonFields& common, size_t slot_size) {
   ctrl_t* ctrl = common.control();
   static constexpr size_t kTwoGroupCapacity = 2 * Group::kWidth - 1;
   if (ABSL_PREDICT_TRUE(capacity <= kTwoGroupCapacity)) {
+    if (IsSmallCapacity(capacity)) return;
     std::memset(ctrl, static_cast<int8_t>(ctrl_t::kEmpty), Group::kWidth);
     std::memset(ctrl + capacity, static_cast<int8_t>(ctrl_t::kEmpty),
                 Group::kWidth);
@@ -449,38 +550,11 @@ void ResetCtrl(CommonFields& common, size_t slot_size) {
   SanitizerPoisonMemoryRegion(common.slot_array(), slot_size * capacity);
 }
 
-// Initializes control bytes for single element table.
-// Capacity of the table must be 1.
-ABSL_ATTRIBUTE_ALWAYS_INLINE inline void InitializeSingleElementControlBytes(
-    uint64_t h2, ctrl_t* new_ctrl) {
-  static constexpr uint64_t kEmptyXorSentinel =
-      static_cast<uint8_t>(ctrl_t::kEmpty) ^
-      static_cast<uint8_t>(ctrl_t::kSentinel);
-  static constexpr uint64_t kEmpty64 = static_cast<uint8_t>(ctrl_t::kEmpty);
-  // The first 8 bytes, where present slot positions are replaced with 0.
-  static constexpr uint64_t kFirstCtrlBytesWithZeroes =
-      k8EmptyBytes ^ kEmpty64 ^ (kEmptyXorSentinel << 8) ^ (kEmpty64 << 16);
-
-  // Fill the original 0th and mirrored 2nd bytes with the hash.
-  // Result will look like:
-  // HSHEEEEE
-  // Where H = h2, E = kEmpty, S = kSentinel.
-  const uint64_t first_ctrl_bytes =
-      (h2 | kFirstCtrlBytesWithZeroes) | (h2 << 16);
-  // Fill last bytes with kEmpty.
-  std::memset(new_ctrl + 1, static_cast<int8_t>(ctrl_t::kEmpty), Group::kWidth);
-  // Overwrite the first 3 bytes with HSH. Other bytes will not be changed.
-  absl::little_endian::Store64(new_ctrl, first_ctrl_bytes);
-}
-
-// Initializes control bytes for growing after SOO to the next capacity.
-// `soo_ctrl` is placed in the position `SooSlotIndex()`.
-// `new_hash` is placed in the position `new_offset`.
-// The table must be non-empty SOO.
-ABSL_ATTRIBUTE_ALWAYS_INLINE inline void
-InitializeThreeElementsControlBytesAfterSoo(ctrl_t soo_ctrl, size_t new_hash,
-                                            size_t new_offset,
-                                            ctrl_t* new_ctrl) {
+// Initializes control bytes for growing from capacity 1 to 3.
+// `orig_h2` is placed in the position `SooSlotIndex()`.
+// `new_h2` is placed in the position `new_offset`.
+ABSL_ATTRIBUTE_ALWAYS_INLINE inline void InitializeThreeElementsControlBytes(
+    h2_t orig_h2, h2_t new_h2, size_t new_offset, ctrl_t* new_ctrl) {
   static constexpr size_t kNewCapacity = NextCapacity(SooCapacity());
   static_assert(kNewCapacity == 3);
   static_assert(is_single_group(kNewCapacity));
@@ -501,9 +575,9 @@ InitializeThreeElementsControlBytesAfterSoo(ctrl_t soo_ctrl, size_t new_hash,
       (kEmptyXorSentinel << (8 * kNewCapacity)) ^
       (kEmpty64 << (8 * kMirroredSooSlotIndex));
 
-  const uint64_t soo_h2 = static_cast<uint64_t>(soo_ctrl);
-  const uint64_t new_h2_xor_empty = static_cast<uint64_t>(
-      H2(new_hash) ^ static_cast<uint8_t>(ctrl_t::kEmpty));
+  const uint64_t soo_h2 = static_cast<uint64_t>(orig_h2);
+  const uint64_t new_h2_xor_empty =
+      static_cast<uint64_t>(new_h2 ^ static_cast<uint8_t>(ctrl_t::kEmpty));
   // Fill the original and mirrored bytes for SOO slot.
   // Result will look like:
   // EHESEHEE
@@ -542,13 +616,44 @@ InitializeThreeElementsControlBytesAfterSoo(ctrl_t soo_ctrl, size_t new_hash,
   // new_ctrl after 2nd store  =      EHNSEHNEEEE
 }
 
+// ClearBackingArrayNoReuse clears the backing array and sets the common
+// fields to the default values for empty non-allocated tables.
+// REQUIRES: c.capacity > policy.soo_capacity.
+void ClearBackingArrayNoReuse(CommonFields& c,
+                              const PolicyFunctions& __restrict policy,
+                              void* alloc) {
+  ABSL_SWISSTABLE_ASSERT(c.capacity() > policy.soo_capacity());
+  // We need to record infoz before calling dealloc, which will unregister
+  // infoz.
+  c.infoz().RecordClearedReservation();
+  c.infoz().RecordStorageChanged(0, policy.soo_capacity());
+  c.infoz().Unregister();
+  (*policy.dealloc)(alloc, c.capacity(), c.control(), policy.slot_size,
+                    policy.slot_align, c.has_infoz());
+  c = policy.soo_enabled ? CommonFields{soo_tag_t{}}
+                         : CommonFields{non_soo_tag_t{}};
+}
+
 }  // namespace
 
-void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size) {
-  ABSL_SWISSTABLE_ASSERT(IsFull(c.control()[index]) &&
-                         "erasing a dangling iterator");
+void EraseMetaOnlySmall(CommonFields& c, bool soo_enabled, size_t slot_size) {
+  ABSL_SWISSTABLE_ASSERT(c.is_small());
+  if (soo_enabled) {
+    c.set_empty_soo();
+    return;
+  }
   c.decrement_size();
   c.infoz().RecordErase();
+  SanitizerPoisonMemoryRegion(c.slot_array(), slot_size);
+}
+
+void EraseMetaOnlyLarge(CommonFields& c, const ctrl_t* ctrl, size_t slot_size) {
+  ABSL_SWISSTABLE_ASSERT(!c.is_small());
+  ABSL_SWISSTABLE_ASSERT(IsFull(*ctrl) && "erasing a dangling iterator");
+  c.decrement_size();
+  c.infoz().RecordErase();
+
+  size_t index = static_cast<size_t>(ctrl - c.control());
 
   if (WasNeverFull(c, index)) {
     SetCtrl(c, index, ctrl_t::kEmpty, slot_size);
@@ -562,22 +667,15 @@ void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size) {
 
 void ClearBackingArray(CommonFields& c,
                        const PolicyFunctions& __restrict policy, void* alloc,
-                       bool reuse, bool soo_enabled) {
+                       bool reuse) {
+  ABSL_SWISSTABLE_ASSERT(c.capacity() > MaxSmallCapacity());
   if (reuse) {
     c.set_size_to_zero();
-    ABSL_SWISSTABLE_ASSERT(!soo_enabled || c.capacity() > SooCapacity());
     ResetCtrl(c, policy.slot_size);
     ResetGrowthLeft(c);
     c.infoz().RecordStorageChanged(0, c.capacity());
   } else {
-    // We need to record infoz before calling dealloc, which will unregister
-    // infoz.
-    c.infoz().RecordClearedReservation();
-    c.infoz().RecordStorageChanged(0, soo_enabled ? SooCapacity() : 0);
-    c.infoz().Unregister();
-    (*policy.dealloc)(alloc, c.capacity(), c.control(), policy.slot_size,
-                      policy.slot_align, c.has_infoz());
-    c = soo_enabled ? CommonFields{soo_tag_t{}} : CommonFields{non_soo_tag_t{}};
+    ClearBackingArrayNoReuse(c, policy, alloc);
   }
 }
 
@@ -599,15 +697,25 @@ size_t FindNewPositionsAndTransferSlots(
   void* new_slots = common.slot_array();
   const void* hash_fn = policy.hash_fn(common);
   const size_t slot_size = policy.slot_size;
+  const size_t seed = common.seed().seed();
 
   const auto insert_slot = [&](void* slot) {
-    size_t hash = policy.hash_slot(hash_fn, slot);
-    auto target = find_first_non_full(common, hash);
-    SetCtrl(common, target.offset, H2(hash), slot_size);
+    size_t hash = policy.hash_slot(hash_fn, slot, seed);
+    FindInfo target;
+    if (common.is_small()) {
+      target = FindInfo{0, 0};
+    } else {
+      target = find_first_non_full(common, hash);
+      SetCtrl(common, target.offset, H2(hash), slot_size);
+    }
     policy.transfer_n(&common, SlotAddress(new_slots, target.offset, slot_size),
                       slot, 1);
     return target.probe_length;
   };
+  if (IsSmallCapacity(old_capacity)) {
+    if (common.size() == 1) insert_slot(old_slots);
+    return 0;
+  }
   size_t total_probe_length = 0;
   for (size_t i = 0; i < old_capacity; ++i) {
     if (IsFull(old_ctrl[i])) {
@@ -618,6 +726,75 @@ size_t FindNewPositionsAndTransferSlots(
   return total_probe_length;
 }
 
+void ReportGrowthToInfozImpl(CommonFields& common, HashtablezInfoHandle infoz,
+                             size_t hash, size_t total_probe_length,
+                             size_t distance_from_desired) {
+  ABSL_SWISSTABLE_ASSERT(infoz.IsSampled());
+  infoz.RecordStorageChanged(common.size() - 1, common.capacity());
+  infoz.RecordRehash(total_probe_length);
+  infoz.RecordInsertMiss(hash, distance_from_desired);
+  common.set_has_infoz();
+  // TODO(b/413062340): we could potentially store infoz in place of the
+  // control pointer for the capacity 1 case.
+  common.set_infoz(infoz);
+}
+
+// Specialization to avoid passing two 0s from hot function.
+ABSL_ATTRIBUTE_NOINLINE void ReportSingleGroupTableGrowthToInfoz(
+    CommonFields& common, HashtablezInfoHandle infoz, size_t hash) {
+  ReportGrowthToInfozImpl(common, infoz, hash, /*total_probe_length=*/0,
+                          /*distance_from_desired=*/0);
+}
+
+ABSL_ATTRIBUTE_NOINLINE void ReportGrowthToInfoz(CommonFields& common,
+                                                 HashtablezInfoHandle infoz,
+                                                 size_t hash,
+                                                 size_t total_probe_length,
+                                                 size_t distance_from_desired) {
+  ReportGrowthToInfozImpl(common, infoz, hash, total_probe_length,
+                          distance_from_desired);
+}
+
+ABSL_ATTRIBUTE_NOINLINE void ReportResizeToInfoz(CommonFields& common,
+                                                 HashtablezInfoHandle infoz,
+                                                 size_t total_probe_length) {
+  ABSL_SWISSTABLE_ASSERT(infoz.IsSampled());
+  infoz.RecordStorageChanged(common.size(), common.capacity());
+  infoz.RecordRehash(total_probe_length);
+  common.set_has_infoz();
+  common.set_infoz(infoz);
+}
+
+struct BackingArrayPtrs {
+  ctrl_t* ctrl;
+  void* slots;
+};
+
+BackingArrayPtrs AllocBackingArray(CommonFields& common,
+                                   const PolicyFunctions& __restrict policy,
+                                   size_t new_capacity, bool has_infoz,
+                                   void* alloc) {
+  RawHashSetLayout layout(new_capacity, policy.slot_size, policy.slot_align,
+                          has_infoz);
+  // Perform a direct call in the common case to allow for profile-guided
+  // heap optimization (PGHO) to understand which allocation function is used.
+  constexpr size_t kDefaultAlignment = BackingArrayAlignment(alignof(size_t));
+  char* mem = static_cast<char*>(
+      ABSL_PREDICT_TRUE(
+          policy.alloc ==
+          (&AllocateBackingArray<kDefaultAlignment, std::allocator<char>>))
+          ? AllocateBackingArray<kDefaultAlignment, std::allocator<char>>(
+                alloc, layout.alloc_size())
+          : policy.alloc(alloc, layout.alloc_size()));
+  const GenerationType old_generation = common.generation();
+  common.set_generation_ptr(
+      reinterpret_cast<GenerationType*>(mem + layout.generation_offset()));
+  common.set_generation(NextGeneration(old_generation));
+
+  return {reinterpret_cast<ctrl_t*>(mem + layout.control_offset()),
+          mem + layout.slot_offset()};
+}
+
 template <ResizeNonSooMode kMode>
 void ResizeNonSooImpl(CommonFields& common,
                       const PolicyFunctions& __restrict policy,
@@ -625,26 +802,25 @@ void ResizeNonSooImpl(CommonFields& common,
   ABSL_SWISSTABLE_ASSERT(IsValidCapacity(new_capacity));
   ABSL_SWISSTABLE_ASSERT(new_capacity > policy.soo_capacity());
 
-  const size_t old_capacity = common.capacity();
-  [[maybe_unused]] ctrl_t* old_ctrl = common.control();
-  [[maybe_unused]] void* old_slots = common.slot_array();
+  [[maybe_unused]] const size_t old_capacity = common.capacity();
+  [[maybe_unused]] ctrl_t* old_ctrl;
+  [[maybe_unused]] void* old_slots;
+  if constexpr (kMode == ResizeNonSooMode::kGuaranteedAllocated) {
+    old_ctrl = common.control();
+    old_slots = common.slot_array();
+  }
 
   const size_t slot_size = policy.slot_size;
-  const size_t slot_align = policy.slot_align;
+  [[maybe_unused]] const size_t slot_align = policy.slot_align;
   const bool has_infoz = infoz.IsSampled();
-
-  common.set_capacity(new_capacity);
-  RawHashSetLayout layout(new_capacity, slot_size, slot_align, has_infoz);
   void* alloc = policy.get_char_alloc(common);
-  char* mem = static_cast<char*>(policy.alloc(alloc, layout.alloc_size()));
-  const GenerationType old_generation = common.generation();
-  common.set_generation_ptr(
-      reinterpret_cast<GenerationType*>(mem + layout.generation_offset()));
-  common.set_generation(NextGeneration(old_generation));
 
-  ctrl_t* new_ctrl = reinterpret_cast<ctrl_t*>(mem + layout.control_offset());
-  common.set_control</*kGenerateSeed=*/true>(new_ctrl);
-  common.set_slots(mem + layout.slot_offset());
+  common.set_capacity(new_capacity);
+  const auto [new_ctrl, new_slots] =
+      AllocBackingArray(common, policy, new_capacity, has_infoz, alloc);
+  common.set_control(new_ctrl);
+  common.set_slots(new_slots);
+  common.generate_new_seed(has_infoz);
 
   size_t total_probe_length = 0;
   ResetCtrl(common, slot_size);
@@ -657,18 +833,19 @@ void ResizeNonSooImpl(CommonFields& common,
         common, policy, old_ctrl, old_slots, old_capacity);
     (*policy.dealloc)(alloc, old_capacity, old_ctrl, slot_size, slot_align,
                       has_infoz);
-    ResetGrowthLeft(GetGrowthInfoFromControl(new_ctrl), new_capacity,
-                    common.size());
+    if (HasGrowthInfoForCapacity(new_capacity)) {
+      ResetGrowthLeft(GetGrowthInfoFromControl(new_ctrl), new_capacity,
+                      common.size());
+    }
   } else {
-    GetGrowthInfoFromControl(new_ctrl).InitGrowthLeftNoDeleted(
-        CapacityToGrowth(new_capacity));
+    if (HasGrowthInfoForCapacity(new_capacity)) {
+      GetGrowthInfoFromControl(new_ctrl).InitGrowthLeftNoDeleted(
+          CapacityToGrowth(new_capacity));
+    }
   }
 
-  if (has_infoz) {
-    common.set_has_infoz();
-    infoz.RecordStorageChanged(common.size(), new_capacity);
-    infoz.RecordRehash(total_probe_length);
-    common.set_infoz(infoz);
+  if (ABSL_PREDICT_FALSE(has_infoz)) {
+    ReportResizeToInfoz(common, infoz, total_probe_length);
   }
 }
 
@@ -697,23 +874,26 @@ void ResizeEmptyNonAllocatedTableImpl(CommonFields& common,
 // It is rare to resize an SOO table with one element to a large size.
 // Requires: `c` contains SOO data.
 void InsertOldSooSlotAndInitializeControlBytes(
-    CommonFields& c, const PolicyFunctions& __restrict policy, size_t hash,
-    ctrl_t* new_ctrl, void* new_slots) {
+    CommonFields& c, const PolicyFunctions& __restrict policy, ctrl_t* new_ctrl,
+    void* new_slots, bool has_infoz) {
   ABSL_SWISSTABLE_ASSERT(c.size() == policy.soo_capacity());
   ABSL_SWISSTABLE_ASSERT(policy.soo_enabled);
   size_t new_capacity = c.capacity();
 
-  c.generate_new_seed();
-  size_t offset = probe(c.seed(), new_capacity, hash).offset();
+  c.generate_new_seed(has_infoz);
+
+  const size_t soo_slot_hash =
+      policy.hash_slot(policy.hash_fn(c), c.soo_data(), c.seed().seed());
+  size_t offset = probe(c.capacity_impl(), soo_slot_hash).offset();
   offset = offset == new_capacity ? 0 : offset;
   SanitizerPoisonMemoryRegion(new_slots, policy.slot_size * new_capacity);
   void* target_slot = SlotAddress(new_slots, offset, policy.slot_size);
   SanitizerUnpoisonMemoryRegion(target_slot, policy.slot_size);
   policy.transfer_n(&c, target_slot, c.soo_data(), 1);
-  c.set_control</*kGenerateSeed=*/false>(new_ctrl);
+  c.set_control(new_ctrl);
   c.set_slots(new_slots);
   ResetCtrl(c, policy.slot_size);
-  SetCtrl(c, offset, H2(hash), policy.slot_size);
+  SetCtrl(c, offset, H2(soo_slot_hash), policy.slot_size);
 }
 
 enum class ResizeFullSooTableSamplingMode {
@@ -739,9 +919,10 @@ void ResizeFullSooTable(CommonFields& common,
                         ResizeFullSooTableSamplingMode sampling_mode) {
   AssertFullSoo(common, policy);
   const size_t slot_size = policy.slot_size;
-  const size_t slot_align = policy.slot_align;
+  void* alloc = policy.get_char_alloc(common);
 
   HashtablezInfoHandle infoz;
+  bool has_infoz = false;
   if (sampling_mode ==
       ResizeFullSooTableSamplingMode::kForceSampleNoResizeIfUnsampled) {
     if (ABSL_PREDICT_FALSE(policy.is_hashtablez_eligible)) {
@@ -749,33 +930,19 @@ void ResizeFullSooTable(CommonFields& common,
                               policy.soo_capacity());
     }
 
-    if (!infoz.IsSampled()) {
-      return;
-    }
+    if (!infoz.IsSampled()) return;
+    has_infoz = true;
   }
 
-  const bool has_infoz = infoz.IsSampled();
-
   common.set_capacity(new_capacity);
 
-  RawHashSetLayout layout(new_capacity, slot_size, slot_align, has_infoz);
-  void* alloc = policy.get_char_alloc(common);
-  char* mem = static_cast<char*>(policy.alloc(alloc, layout.alloc_size()));
-  const GenerationType old_generation = common.generation();
-  common.set_generation_ptr(
-      reinterpret_cast<GenerationType*>(mem + layout.generation_offset()));
-  common.set_generation(NextGeneration(old_generation));
-
   // We do not set control and slots in CommonFields yet to avoid overriding
   // SOO data.
-  ctrl_t* new_ctrl = reinterpret_cast<ctrl_t*>(mem + layout.control_offset());
-  void* new_slots = mem + layout.slot_offset();
-
-  const size_t soo_slot_hash =
-      policy.hash_slot(policy.hash_fn(common), common.soo_data());
+  const auto [new_ctrl, new_slots] =
+      AllocBackingArray(common, policy, new_capacity, has_infoz, alloc);
 
-  InsertOldSooSlotAndInitializeControlBytes(common, policy, soo_slot_hash,
-                                            new_ctrl, new_slots);
+  InsertOldSooSlotAndInitializeControlBytes(common, policy, new_ctrl, new_slots,
+                                            has_infoz);
   ResetGrowthLeft(common);
   if (has_infoz) {
     common.set_has_infoz();
@@ -853,7 +1020,7 @@ void GrowIntoSingleGroupShuffleControlBytes(ctrl_t* __restrict old_ctrl,
     return;
   }
 
-  ABSL_SWISSTABLE_ASSERT(Group::kWidth == 16);
+  ABSL_SWISSTABLE_ASSERT(Group::kWidth == 16);  // NOLINT(misc-static-assert)
 
   // Fill the second half of the main control bytes with kEmpty.
   // For small capacity that may write into mirrored control bytes.
@@ -920,7 +1087,7 @@ template <typename ProbedItem>
 ABSL_ATTRIBUTE_NOINLINE size_t DecodeAndInsertImpl(
     CommonFields& c, const PolicyFunctions& __restrict policy,
     const ProbedItem* start, const ProbedItem* end, void* old_slots) {
-  const size_t new_capacity = c.capacity();
+  const HashtableCapacity new_capacity = c.capacity_impl();
 
   void* new_slots = c.slot_array();
   ctrl_t* new_ctrl = c.control();
@@ -935,8 +1102,8 @@ ABSL_ATTRIBUTE_NOINLINE size_t DecodeAndInsertImpl(
     total_probe_length += target.probe_length;
     const size_t old_index = static_cast<size_t>(start->source_offset);
     const size_t new_i = target.offset;
-    ABSL_SWISSTABLE_ASSERT(old_index < new_capacity / 2);
-    ABSL_SWISSTABLE_ASSERT(new_i < new_capacity);
+    ABSL_SWISSTABLE_ASSERT(old_index < new_capacity.capacity() / 2);
+    ABSL_SWISSTABLE_ASSERT(new_i < new_capacity.capacity());
     ABSL_SWISSTABLE_ASSERT(IsEmpty(new_ctrl[new_i]));
     void* src_slot = SlotAddress(old_slots, old_index, slot_size);
     void* dst_slot = SlotAddress(new_slots, new_i, slot_size);
@@ -965,12 +1132,13 @@ ABSL_ATTRIBUTE_NOINLINE size_t ProcessProbedMarkedElements(
   const void* hash_fn = policy.hash_fn(c);
   auto hash_slot = policy.hash_slot;
   auto transfer_n = policy.transfer_n;
+  const size_t seed = c.seed().seed();
   for (size_t old_index = start; old_index < old_capacity; ++old_index) {
     if (old_ctrl[old_index] != ctrl_t::kSentinel) {
       continue;
     }
     void* src_slot = SlotAddress(old_slots, old_index, slot_size);
-    const size_t hash = hash_slot(hash_fn, src_slot);
+    const size_t hash = hash_slot(hash_fn, src_slot, seed);
     const FindInfo target = find_first_non_full(c, hash);
     total_probe_length += target.probe_length;
     const size_t new_i = target.offset;
@@ -1062,7 +1230,8 @@ class ProbedItemEncoder {
 
   ProbedItem* OverflowBufferStart() const {
     // We reuse GrowthInfo memory as well.
-    return AlignToNextItem(control_ - ControlOffset(/*has_infoz=*/false));
+    return AlignToNextItem(control_ - ControlOffset(/*has_infoz=*/false,
+                                                    /*has_growth_info=*/true));
   }
 
   // Encodes item when previously allocated buffer is full.
@@ -1224,6 +1393,79 @@ size_t GrowToNextCapacityDispatch(CommonFields& common,
   }
 }
 
+void IncrementSmallSizeNonSoo(CommonFields& common,
+                              const PolicyFunctions& __restrict policy) {
+  ABSL_SWISSTABLE_ASSERT(common.is_small());
+  common.increment_size();
+  SanitizerUnpoisonMemoryRegion(common.slot_array(), policy.slot_size);
+}
+
+void IncrementSmallSize(CommonFields& common,
+                        const PolicyFunctions& __restrict policy) {
+  ABSL_SWISSTABLE_ASSERT(common.is_small());
+  if (policy.soo_enabled) {
+    common.set_full_soo();
+  } else {
+    IncrementSmallSizeNonSoo(common, policy);
+  }
+}
+
+std::pair<ctrl_t*, void*> Grow1To3AndPrepareInsert(
+    CommonFields& common, const PolicyFunctions& __restrict policy,
+    absl::FunctionRef<size_t(size_t)> get_hash) {
+  // TODO(b/413062340): Refactor to reuse more code with
+  // GrowSooTableToNextCapacityAndPrepareInsert.
+  ABSL_SWISSTABLE_ASSERT(common.capacity() == 1);
+  ABSL_SWISSTABLE_ASSERT(!common.empty());
+  ABSL_SWISSTABLE_ASSERT(!policy.soo_enabled);
+  constexpr size_t kOldCapacity = 1;
+  constexpr size_t kNewCapacity = NextCapacity(kOldCapacity);
+  ctrl_t* old_ctrl = common.control();
+  void* old_slots = common.slot_array();
+
+  const size_t slot_size = policy.slot_size;
+  const size_t slot_align = policy.slot_align;
+  void* alloc = policy.get_char_alloc(common);
+  HashtablezInfoHandle infoz = common.infoz();
+  const bool has_infoz = infoz.IsSampled();
+  common.set_capacity(kNewCapacity);
+
+  const auto [new_ctrl, new_slots] =
+      AllocBackingArray(common, policy, kNewCapacity, has_infoz, alloc);
+  common.set_control(new_ctrl);
+  common.set_slots(new_slots);
+  SanitizerPoisonMemoryRegion(new_slots, kNewCapacity * slot_size);
+
+  if (ABSL_PREDICT_TRUE(!has_infoz)) {
+    // When we're sampled, we already have a seed.
+    common.generate_new_seed(/*has_infoz=*/false);
+  }
+  const size_t new_hash = get_hash(common.seed().seed());
+  h2_t new_h2 = H2(new_hash);
+  size_t orig_hash =
+      policy.hash_slot(policy.hash_fn(common), old_slots, common.seed().seed());
+  size_t offset = Resize1To3NewOffset(new_hash, common.seed());
+  InitializeThreeElementsControlBytes(H2(orig_hash), new_h2, offset, new_ctrl);
+
+  void* old_element_target = NextSlot(new_slots, slot_size);
+  SanitizerUnpoisonMemoryRegion(old_element_target, slot_size);
+  policy.transfer_n(&common, old_element_target, old_slots, 1);
+
+  void* new_element_target_slot = SlotAddress(new_slots, offset, slot_size);
+  SanitizerUnpoisonMemoryRegion(new_element_target_slot, slot_size);
+
+  policy.dealloc(alloc, kOldCapacity, old_ctrl, slot_size, slot_align,
+                 has_infoz);
+  PrepareInsertCommon(common);
+  ABSL_SWISSTABLE_ASSERT(common.size() == 2);
+  GetGrowthInfoFromControl(new_ctrl).InitGrowthLeftNoDeleted(kNewCapacity - 2);
+
+  if (ABSL_PREDICT_FALSE(has_infoz)) {
+    ReportSingleGroupTableGrowthToInfoz(common, infoz, new_hash);
+  }
+  return {new_ctrl + offset, new_element_target_slot};
+}
+
 // Grows to next capacity and prepares insert for the given new_hash.
 // Returns the offset of the new element.
 size_t GrowToNextCapacityAndPrepareInsert(
@@ -1231,98 +1473,120 @@ size_t GrowToNextCapacityAndPrepareInsert(
     size_t new_hash) {
   ABSL_SWISSTABLE_ASSERT(common.growth_left() == 0);
   const size_t old_capacity = common.capacity();
-  ABSL_SWISSTABLE_ASSERT(old_capacity == 0 ||
-                         old_capacity > policy.soo_capacity());
+  ABSL_SWISSTABLE_ASSERT(old_capacity > policy.soo_capacity());
+  ABSL_SWISSTABLE_ASSERT(!IsSmallCapacity(old_capacity));
 
   const size_t new_capacity = NextCapacity(old_capacity);
-  ABSL_SWISSTABLE_ASSERT(IsValidCapacity(new_capacity));
-  ABSL_SWISSTABLE_ASSERT(new_capacity > policy.soo_capacity());
-
   ctrl_t* old_ctrl = common.control();
   void* old_slots = common.slot_array();
 
   common.set_capacity(new_capacity);
   const size_t slot_size = policy.slot_size;
   const size_t slot_align = policy.slot_align;
-  HashtablezInfoHandle infoz;
-  if (old_capacity > 0) {
-    infoz = common.infoz();
-  } else {
-    const bool should_sample =
-        policy.is_hashtablez_eligible && ShouldSampleNextTable();
-    if (ABSL_PREDICT_FALSE(should_sample)) {
-      infoz = ForcedTrySample(slot_size, policy.key_size, policy.value_size,
-                              policy.soo_capacity());
-    }
-  }
-  const bool has_infoz = infoz.IsSampled();
-
-  RawHashSetLayout layout(new_capacity, slot_size, slot_align, has_infoz);
   void* alloc = policy.get_char_alloc(common);
-  char* mem = static_cast<char*>(policy.alloc(alloc, layout.alloc_size()));
-  const GenerationType old_generation = common.generation();
-  common.set_generation_ptr(
-      reinterpret_cast<GenerationType*>(mem + layout.generation_offset()));
-  common.set_generation(NextGeneration(old_generation));
+  HashtablezInfoHandle infoz = common.infoz();
+  const bool has_infoz = infoz.IsSampled();
 
-  ctrl_t* new_ctrl = reinterpret_cast<ctrl_t*>(mem + layout.control_offset());
-  void* new_slots = mem + layout.slot_offset();
-  common.set_control</*kGenerateSeed=*/false>(new_ctrl);
+  const auto [new_ctrl, new_slots] =
+      AllocBackingArray(common, policy, new_capacity, has_infoz, alloc);
+  common.set_control(new_ctrl);
   common.set_slots(new_slots);
   SanitizerPoisonMemoryRegion(new_slots, new_capacity * slot_size);
 
   h2_t new_h2 = H2(new_hash);
   size_t total_probe_length = 0;
   FindInfo find_info;
-  if (old_capacity == 0) {
-    static_assert(NextCapacity(0) == 1);
-    InitializeSingleElementControlBytes(new_h2, new_ctrl);
-    common.generate_new_seed();
-    find_info = FindInfo{0, 0};
-    SanitizerUnpoisonMemoryRegion(new_slots, slot_size);
+  if (ABSL_PREDICT_TRUE(is_single_group(new_capacity))) {
+    size_t offset;
+    GrowIntoSingleGroupShuffleControlBytes(old_ctrl, old_capacity, new_ctrl,
+                                           new_capacity);
+    // We put the new element either at the beginning or at the end of the
+    // table with approximately equal probability.
+    offset =
+        SingleGroupTableH1(new_hash, common.seed()) & 1 ? 0 : new_capacity - 1;
+
+    ABSL_SWISSTABLE_ASSERT(IsEmpty(new_ctrl[offset]));
+    SetCtrlInSingleGroupTable(common, offset, new_h2, policy.slot_size);
+    find_info = FindInfo{offset, 0};
+    // Single group tables have all slots full on resize. So we can transfer
+    // all slots without checking the control bytes.
+    ABSL_SWISSTABLE_ASSERT(common.size() == old_capacity);
+    void* target = NextSlot(new_slots, slot_size);
+    SanitizerUnpoisonMemoryRegion(target, old_capacity * slot_size);
+    policy.transfer_n(&common, target, old_slots, old_capacity);
   } else {
-    if (ABSL_PREDICT_TRUE(is_single_group(new_capacity))) {
-      GrowIntoSingleGroupShuffleControlBytes(old_ctrl, old_capacity, new_ctrl,
-                                             new_capacity);
-      // Single group tables have all slots full on resize. So we can transfer
-      // all slots without checking the control bytes.
-      ABSL_SWISSTABLE_ASSERT(common.size() == old_capacity);
-      auto* target = NextSlot(new_slots, slot_size);
-      SanitizerUnpoisonMemoryRegion(target, old_capacity * slot_size);
-      policy.transfer_n(&common, target, old_slots, old_capacity);
-      // We put the new element either at the beginning or at the end of the
-      // table with approximately equal probability.
-      size_t offset = SingleGroupTableH1(new_hash, common.seed()) & 1
-                          ? 0
-                          : new_capacity - 1;
-
-      ABSL_SWISSTABLE_ASSERT(IsEmpty(new_ctrl[offset]));
-      SetCtrlInSingleGroupTable(common, offset, new_h2, policy.slot_size);
-      find_info = FindInfo{offset, 0};
-    } else {
-      total_probe_length =
-          GrowToNextCapacityDispatch(common, policy, old_ctrl, old_slots);
-      find_info = find_first_non_full(common, new_hash);
-      SetCtrlInLargeTable(common, find_info.offset, new_h2, policy.slot_size);
-    }
-    ABSL_SWISSTABLE_ASSERT(old_capacity > policy.soo_capacity());
-    (*policy.dealloc)(alloc, old_capacity, old_ctrl, slot_size, slot_align,
-                      has_infoz);
+    total_probe_length =
+        GrowToNextCapacityDispatch(common, policy, old_ctrl, old_slots);
+    find_info = find_first_non_full(common, new_hash);
+    SetCtrlInLargeTable(common, find_info.offset, new_h2, policy.slot_size);
   }
+  ABSL_SWISSTABLE_ASSERT(old_capacity > policy.soo_capacity());
+  (*policy.dealloc)(alloc, old_capacity, old_ctrl, slot_size, slot_align,
+                    has_infoz);
   PrepareInsertCommon(common);
   ResetGrowthLeft(GetGrowthInfoFromControl(new_ctrl), new_capacity,
                   common.size());
 
   if (ABSL_PREDICT_FALSE(has_infoz)) {
-    common.set_has_infoz();
-    infoz.RecordStorageChanged(common.size() - 1, new_capacity);
-    infoz.RecordRehash(total_probe_length);
-    infoz.RecordInsert(new_hash, find_info.probe_length);
-    common.set_infoz(infoz);
+    ReportGrowthToInfoz(common, infoz, new_hash, total_probe_length,
+                        find_info.probe_length);
   }
   return find_info.offset;
 }
 
+}  // namespace
+
+std::pair<ctrl_t*, void*> PrepareInsertSmallNonSoo(
+    CommonFields& common, const PolicyFunctions& __restrict policy,
+    absl::FunctionRef<size_t(size_t)> get_hash) {
+  ABSL_SWISSTABLE_ASSERT(common.is_small());
+  ABSL_SWISSTABLE_ASSERT(!policy.soo_enabled);
+  if (common.capacity() == 1) {
+    if (common.empty()) {
+      IncrementSmallSizeNonSoo(common, policy);
+      if (common.has_infoz()) {
+        common.infoz().RecordInsertMiss(get_hash(common.seed().seed()),
+                                        /*distance_from_desired=*/0);
+      }
+      return {SooControl(), common.slot_array()};
+    } else {
+      return Grow1To3AndPrepareInsert(common, policy, get_hash);
+    }
+  }
+
+  // Growing from 0 to 1 capacity.
+  ABSL_SWISSTABLE_ASSERT(common.capacity() == 0);
+  constexpr size_t kNewCapacity = 1;
+
+  common.set_capacity(kNewCapacity);
+  HashtablezInfoHandle infoz;
+  const bool should_sample =
+      policy.is_hashtablez_eligible && ShouldSampleNextTable();
+  if (ABSL_PREDICT_FALSE(should_sample)) {
+    infoz = ForcedTrySample(policy.slot_size, policy.key_size,
+                            policy.value_size, policy.soo_capacity());
+  }
+  const bool has_infoz = infoz.IsSampled();
+  void* alloc = policy.get_char_alloc(common);
+
+  const auto [new_ctrl, new_slots] =
+      AllocBackingArray(common, policy, kNewCapacity, has_infoz, alloc);
+  common.set_control(new_ctrl);
+  common.set_slots(new_slots);
+
+  static_assert(NextCapacity(0) == 1);
+  PrepareInsertCommon(common);
+
+  if (ABSL_PREDICT_FALSE(has_infoz)) {
+    common.generate_new_seed(/*has_infoz=*/true);
+    ReportSingleGroupTableGrowthToInfoz(common, infoz,
+                                        get_hash(common.seed().seed()));
+  }
+  return {SooControl(), new_slots};
+}
+
+namespace {
+
 // Called whenever the table needs to vacate empty slots either by removing
 // tombstones via rehash or growth to next capacity.
 ABSL_ATTRIBUTE_NOINLINE
@@ -1382,11 +1646,11 @@ size_t RehashOrGrowToNextCapacityAndPrepareInsert(
   }
 }
 
-// Slow path for PrepareInsertNonSoo that is called when the table has deleted
+// Slow path for PrepareInsertLarge that is called when the table has deleted
 // slots or need to be resized or rehashed.
-size_t PrepareInsertNonSooSlow(CommonFields& common,
-                               const PolicyFunctions& __restrict policy,
-                               size_t hash) {
+size_t PrepareInsertLargeSlow(CommonFields& common,
+                              const PolicyFunctions& __restrict policy,
+                              size_t hash) {
   const GrowthInfo growth_info = common.growth_info();
   ABSL_SWISSTABLE_ASSERT(!growth_info.HasNoDeletedAndGrowthLeft());
   if (ABSL_PREDICT_TRUE(growth_info.HasNoGrowthLeftAndNoDeleted())) {
@@ -1404,7 +1668,7 @@ size_t PrepareInsertNonSooSlow(CommonFields& common,
   PrepareInsertCommon(common);
   common.growth_info().OverwriteControlAsFull(common.control()[target.offset]);
   SetCtrlInLargeTable(common, target.offset, H2(hash), policy.slot_size);
-  common.infoz().RecordInsert(hash, target.probe_length);
+  common.infoz().RecordInsertMiss(hash, target.probe_length);
   return target.offset;
 }
 
@@ -1417,14 +1681,15 @@ size_t PrepareInsertNonSooSlow(CommonFields& common,
 ABSL_ATTRIBUTE_NOINLINE size_t
 GrowEmptySooTableToNextCapacityForceSamplingAndPrepareInsert(
     CommonFields& common, const PolicyFunctions& __restrict policy,
-    size_t new_hash) {
+    absl::FunctionRef<size_t(size_t)> get_hash) {
   ResizeEmptyNonAllocatedTableImpl(common, policy, NextCapacity(SooCapacity()),
                                    /*force_infoz=*/true);
   PrepareInsertCommon(common);
   common.growth_info().OverwriteEmptyAsFull();
+  const size_t new_hash = get_hash(common.seed().seed());
   SetCtrlInSingleGroupTable(common, SooSlotIndex(), H2(new_hash),
                             policy.slot_size);
-  common.infoz().RecordInsert(new_hash, /*distance_from_desired=*/0);
+  common.infoz().RecordInsertMiss(new_hash, /*distance_from_desired=*/0);
   return SooSlotIndex();
 }
 
@@ -1437,7 +1702,7 @@ GrowEmptySooTableToNextCapacityForceSamplingAndPrepareInsert(
 void ReserveEmptyNonAllocatedTableToFitNewSize(
     CommonFields& common, const PolicyFunctions& __restrict policy,
     size_t new_size) {
-  ValidateMaxSize(new_size, policy.slot_size);
+  ValidateMaxSize(new_size, policy.key_size, policy.slot_size);
   ABSL_ASSUME(new_size > 0);
   ResizeEmptyNonAllocatedTableImpl(common, policy, SizeToCapacity(new_size),
                                    /*force_infoz=*/false);
@@ -1456,7 +1721,7 @@ ABSL_ATTRIBUTE_NOINLINE void ReserveAllocatedTable(
     CommonFields& common, const PolicyFunctions& __restrict policy,
     size_t new_size) {
   const size_t cap = common.capacity();
-  ValidateMaxSize(new_size, policy.slot_size);
+  ValidateMaxSize(new_size, policy.key_size, policy.slot_size);
   ABSL_ASSUME(new_size > 0);
   const size_t new_capacity = SizeToCapacity(new_size);
   if (cap == policy.soo_capacity()) {
@@ -1471,6 +1736,17 @@ ABSL_ATTRIBUTE_NOINLINE void ReserveAllocatedTable(
   common.infoz().RecordReservation(new_size);
 }
 
+// As `ResizeFullSooTableToNextCapacity`, except that we also force the SOO
+// table to be sampled. SOO tables need to switch from SOO to heap in order to
+// store the infoz. No-op if sampling is disabled or not possible.
+void GrowFullSooTableToNextCapacityForceSampling(
+    CommonFields& common, const PolicyFunctions& __restrict policy) {
+  AssertFullSoo(common, policy);
+  ResizeFullSooTable(
+      common, policy, NextCapacity(SooCapacity()),
+      ResizeFullSooTableSamplingMode::kForceSampleNoResizeIfUnsampled);
+}
+
 }  // namespace
 
 void* GetRefForEmptyClass(CommonFields& common) {
@@ -1492,7 +1768,7 @@ void ReserveEmptyNonAllocatedTableToFitBucketCount(
     CommonFields& common, const PolicyFunctions& __restrict policy,
     size_t bucket_count) {
   size_t new_capacity = NormalizeCapacity(bucket_count);
-  ValidateMaxSize(CapacityToGrowth(new_capacity), policy.slot_size);
+  ValidateMaxCapacity(new_capacity, policy.key_size, policy.slot_size);
   ResizeEmptyNonAllocatedTableImpl(common, policy, new_capacity,
                                    /*force_infoz=*/false);
 }
@@ -1501,46 +1777,38 @@ void ReserveEmptyNonAllocatedTableToFitBucketCount(
 template <size_t SooSlotMemcpySize, bool TransferUsesMemcpy>
 size_t GrowSooTableToNextCapacityAndPrepareInsert(
     CommonFields& common, const PolicyFunctions& __restrict policy,
-    size_t new_hash, ctrl_t soo_slot_ctrl) {
+    absl::FunctionRef<size_t(size_t)> get_hash, bool force_sampling) {
   AssertSoo(common, policy);
-  if (ABSL_PREDICT_FALSE(soo_slot_ctrl == ctrl_t::kEmpty)) {
+  if (ABSL_PREDICT_FALSE(force_sampling)) {
     // The table is empty, it is only used for forced sampling of SOO tables.
     return GrowEmptySooTableToNextCapacityForceSamplingAndPrepareInsert(
-        common, policy, new_hash);
+        common, policy, get_hash);
   }
   ABSL_SWISSTABLE_ASSERT(common.size() == policy.soo_capacity());
   static constexpr size_t kNewCapacity = NextCapacity(SooCapacity());
   const size_t slot_size = policy.slot_size;
-  const size_t slot_align = policy.slot_align;
+  void* alloc = policy.get_char_alloc(common);
   common.set_capacity(kNewCapacity);
 
   // Since the table is not empty, it will not be sampled.
   // The decision to sample was already made during the first insertion.
-  RawHashSetLayout layout(kNewCapacity, slot_size, slot_align,
-                          /*has_infoz=*/false);
-  void* alloc = policy.get_char_alloc(common);
-  char* mem = static_cast<char*>(policy.alloc(alloc, layout.alloc_size()));
-  const GenerationType old_generation = common.generation();
-  common.set_generation_ptr(
-      reinterpret_cast<GenerationType*>(mem + layout.generation_offset()));
-  common.set_generation(NextGeneration(old_generation));
-
+  //
   // We do not set control and slots in CommonFields yet to avoid overriding
   // SOO data.
-  ctrl_t* new_ctrl = reinterpret_cast<ctrl_t*>(mem + layout.control_offset());
-  void* new_slots = mem + layout.slot_offset();
+  const auto [new_ctrl, new_slots] = AllocBackingArray(
+      common, policy, kNewCapacity, /*has_infoz=*/false, alloc);
 
   PrepareInsertCommon(common);
   ABSL_SWISSTABLE_ASSERT(common.size() == 2);
   GetGrowthInfoFromControl(new_ctrl).InitGrowthLeftNoDeleted(kNewCapacity - 2);
-  common.generate_new_seed();
+  common.generate_new_seed(/*has_infoz=*/false);
+  const h2_t soo_slot_h2 = H2(policy.hash_slot(
+      policy.hash_fn(common), common.soo_data(), common.seed().seed()));
+  const size_t new_hash = get_hash(common.seed().seed());
 
-  // After resize from capacity 1 to 3, we always have exactly the slot with
-  // index 1 occupied, so we need to insert either at index 0 or index 2.
-  static_assert(SooSlotIndex() == 1);
-  const size_t offset = SingleGroupTableH1(new_hash, common.seed()) & 2;
-  InitializeThreeElementsControlBytesAfterSoo(soo_slot_ctrl, new_hash, offset,
-                                              new_ctrl);
+  const size_t offset = Resize1To3NewOffset(new_hash, common.seed());
+  InitializeThreeElementsControlBytes(soo_slot_h2, H2(new_hash), offset,
+                                      new_ctrl);
 
   SanitizerPoisonMemoryRegion(new_slots, slot_size * kNewCapacity);
   void* target_slot = SlotAddress(new_slots, SooSlotIndex(), slot_size);
@@ -1562,31 +1830,23 @@ size_t GrowSooTableToNextCapacityAndPrepareInsert(
     static_assert(SooSlotMemcpySize == 0);
     policy.transfer_n(&common, target_slot, common.soo_data(), 1);
   }
-  // Seed was already generated above.
-  common.set_control</*kGenerateSeed=*/false>(new_ctrl);
+  common.set_control(new_ctrl);
   common.set_slots(new_slots);
 
-  common.infoz().RecordInsert(new_hash, /*distance_from_desired=*/0);
+  // Full SOO table couldn't be sampled. If SOO table is sampled, it would
+  // have been resized to the next capacity.
+  ABSL_SWISSTABLE_ASSERT(!common.infoz().IsSampled());
   SanitizerUnpoisonMemoryRegion(SlotAddress(new_slots, offset, slot_size),
                                 slot_size);
   return offset;
 }
 
-void GrowFullSooTableToNextCapacityForceSampling(
-    CommonFields& common, const PolicyFunctions& __restrict policy) {
-  AssertFullSoo(common, policy);
-  ResizeFullSooTable(
-      common, policy, NextCapacity(SooCapacity()),
-      ResizeFullSooTableSamplingMode::kForceSampleNoResizeIfUnsampled);
-}
-
 void Rehash(CommonFields& common, const PolicyFunctions& __restrict policy,
             size_t n) {
   const size_t cap = common.capacity();
 
   auto clear_backing_array = [&]() {
-    ClearBackingArray(common, policy, policy.get_char_alloc(common),
-                      /*reuse=*/false, policy.soo_enabled);
+    ClearBackingArrayNoReuse(common, policy, policy.get_char_alloc(common));
   };
 
   const size_t slot_size = policy.slot_size;
@@ -1612,7 +1872,7 @@ void Rehash(CommonFields& common, const PolicyFunctions& __restrict policy,
       }
       ABSL_SWISSTABLE_ASSERT(slot_size <= sizeof(HeapOrSoo));
       ABSL_SWISSTABLE_ASSERT(policy.slot_align <= alignof(HeapOrSoo));
-      HeapOrSoo tmp_slot(uninitialized_tag_t{});
+      HeapOrSoo tmp_slot;
       size_t begin_offset = FindFirstFullSlot(0, cap, common.control());
       policy.transfer_n(
           &common, &tmp_slot,
@@ -1624,11 +1884,11 @@ void Rehash(CommonFields& common, const PolicyFunctions& __restrict policy,
     }
   }
 
-  ValidateMaxSize(n, policy.slot_size);
   // bitor is a faster way of doing `max` here. We will round up to the next
   // power-of-2-minus-1, so bitor is good enough.
   const size_t new_capacity =
       NormalizeCapacity(n | SizeToCapacity(common.size()));
+  ValidateMaxCapacity(new_capacity, policy.key_size, policy.slot_size);
   // n == 0 unconditionally rehashes as per the standard.
   if (n == 0 || new_capacity > cap) {
     if (cap == policy.soo_capacity()) {
@@ -1655,19 +1915,22 @@ void Copy(CommonFields& common, const PolicyFunctions& __restrict policy,
   ABSL_SWISSTABLE_ASSERT(size > 0);
   const size_t soo_capacity = policy.soo_capacity();
   const size_t slot_size = policy.slot_size;
-  if (size <= soo_capacity) {
-    ABSL_SWISSTABLE_ASSERT(size == 1);
-    common.set_full_soo();
+  const bool soo_enabled = policy.soo_enabled;
+  if (size == 1) {
+    if (!soo_enabled) ReserveTableToFitNewSize(common, policy, 1);
+    IncrementSmallSize(common, policy);
+    const size_t other_capacity = other.capacity();
     const void* other_slot =
-        other.capacity() <= soo_capacity
-            ? other.soo_data()
-            : SlotAddress(
-                  other.slot_array(),
-                  FindFirstFullSlot(0, other.capacity(), other.control()),
-                  slot_size);
-    copy_fn(common.soo_data(), other_slot);
-
-    if (policy.is_hashtablez_eligible && ShouldSampleNextTable()) {
+        other_capacity <= soo_capacity ? other.soo_data()
+        : other.is_small()
+            ? other.slot_array()
+            : SlotAddress(other.slot_array(),
+                          FindFirstFullSlot(0, other_capacity, other.control()),
+                          slot_size);
+    copy_fn(soo_enabled ? common.soo_data() : common.slot_array(), other_slot);
+
+    if (soo_enabled && policy.is_hashtablez_eligible &&
+        ShouldSampleNextTable()) {
       GrowFullSooTableToNextCapacityForceSampling(common, policy);
     }
     return;
@@ -1678,47 +1941,22 @@ void Copy(CommonFields& common, const PolicyFunctions& __restrict policy,
   ABSL_SWISSTABLE_ASSERT(other.capacity() > soo_capacity);
   const size_t cap = common.capacity();
   ABSL_SWISSTABLE_ASSERT(cap > soo_capacity);
-  // Note about single group tables:
-  // 1. It is correct to have any order of elements.
-  // 2. Order has to be non deterministic.
-  // 3. We are assigning elements with arbitrary `shift` starting from
-  //    `capacity + shift` position.
-  // 4. `shift` must be coprime with `capacity + 1` in order to be able to use
-  //     modular arithmetic to traverse all positions, instead of cycling
-  //     through a subset of positions. Odd numbers are coprime with any
-  //     `capacity + 1` (2^N).
   size_t offset = cap;
-  const size_t shift = is_single_group(cap) ? (common.seed().seed() | 1) : 0;
   const void* hash_fn = policy.hash_fn(common);
   auto hasher = policy.hash_slot;
+  const size_t seed = common.seed().seed();
   IterateOverFullSlotsImpl(
-      other, slot_size, [&](const ctrl_t* that_ctrl, void* that_slot) {
-        if (shift == 0) {
-          // Big tables case. Position must be searched via probing.
-          // The table is guaranteed to be empty, so we can do faster than
-          // a full `insert`.
-          const size_t hash = (*hasher)(hash_fn, that_slot);
-          FindInfo target = find_first_non_full(common, hash);
-          infoz.RecordInsert(hash, target.probe_length);
-          offset = target.offset;
-        } else {
-          // Small tables case. Next position is computed via shift.
-          offset = (offset + shift) & cap;
-        }
-        const h2_t h2 = static_cast<h2_t>(*that_ctrl);
-        // We rely on the hash not changing for small tables.
-        ABSL_SWISSTABLE_ASSERT(
-            H2((*hasher)(hash_fn, that_slot)) == h2 &&
-            "hash function value changed unexpectedly during the copy");
-        SetCtrl(common, offset, h2, slot_size);
+      other, slot_size, [&](const ctrl_t*, void* that_slot) {
+        // The table is guaranteed to be empty, so we can do faster than
+        // a full `insert`.
+        const size_t hash = (*hasher)(hash_fn, that_slot, seed);
+        FindInfo target = find_first_non_full(common, hash);
+        infoz.RecordInsertMiss(hash, target.probe_length);
+        offset = target.offset;
+        SetCtrl(common, offset, H2(hash), slot_size);
         copy_fn(SlotAddress(common.slot_array(), offset, slot_size), that_slot);
         common.maybe_increment_generation_on_insert();
       });
-  if (shift != 0) {
-    // On small table copy we do not record individual inserts.
-    // RecordInsert requires hash, but it is unknown for small tables.
-    infoz.RecordStorageChanged(size, cap);
-  }
   common.increment_size(size);
   common.growth_info().OverwriteManyEmptyAsFull(size);
 }
@@ -1737,41 +1975,62 @@ void ReserveTableToFitNewSize(CommonFields& common,
   ABSL_SWISSTABLE_ASSERT(!common.empty() || cap > policy.soo_capacity());
   ABSL_SWISSTABLE_ASSERT(cap > 0);
   const size_t max_size_before_growth =
-      cap <= policy.soo_capacity() ? policy.soo_capacity()
-                                   : common.size() + common.growth_left();
+      IsSmallCapacity(cap) ? cap : common.size() + common.growth_left();
   if (new_size <= max_size_before_growth) {
     return;
   }
   ReserveAllocatedTable(common, policy, new_size);
 }
 
-size_t PrepareInsertNonSoo(CommonFields& common,
-                           const PolicyFunctions& __restrict policy,
-                           size_t hash, FindInfo target) {
-  const bool rehash_for_bug_detection =
-      common.should_rehash_for_bug_detection_on_insert() &&
-      // Required to allow use of ResizeAllocatedTable.
-      common.capacity() > 0;
-  if (rehash_for_bug_detection) {
-    // Move to a different heap allocation in order to detect bugs.
-    const size_t cap = common.capacity();
-    ResizeAllocatedTableWithSeedChange(
-        common, policy, common.growth_left() > 0 ? cap : NextCapacity(cap));
-    target = find_first_non_full(common, hash);
-  }
-
+namespace {
+size_t PrepareInsertLargeImpl(CommonFields& common,
+                              const PolicyFunctions& __restrict policy,
+                              size_t hash,
+                              Group::NonIterableBitMaskType mask_empty,
+                              FindInfo target_group) {
+  ABSL_SWISSTABLE_ASSERT(!common.is_small());
   const GrowthInfo growth_info = common.growth_info();
   // When there are no deleted slots in the table
   // and growth_left is positive, we can insert at the first
   // empty slot in the probe sequence (target).
   if (ABSL_PREDICT_FALSE(!growth_info.HasNoDeletedAndGrowthLeft())) {
-    return PrepareInsertNonSooSlow(common, policy, hash);
+    return PrepareInsertLargeSlow(common, policy, hash);
   }
   PrepareInsertCommon(common);
   common.growth_info().OverwriteEmptyAsFull();
-  SetCtrl(common, target.offset, H2(hash), policy.slot_size);
-  common.infoz().RecordInsert(hash, target.probe_length);
-  return target.offset;
+  target_group.offset += mask_empty.LowestBitSet();
+  target_group.offset &= common.capacity();
+  SetCtrl(common, target_group.offset, H2(hash), policy.slot_size);
+  common.infoz().RecordInsertMiss(hash, target_group.probe_length);
+  return target_group.offset;
+}
+}  // namespace
+
+size_t PrepareInsertLarge(CommonFields& common,
+                          const PolicyFunctions& __restrict policy, size_t hash,
+                          Group::NonIterableBitMaskType mask_empty,
+                          FindInfo target_group) {
+  // NOLINTNEXTLINE(misc-static-assert)
+  ABSL_SWISSTABLE_ASSERT(!SwisstableGenerationsEnabled());
+  return PrepareInsertLargeImpl(common, policy, hash, mask_empty, target_group);
+}
+
+size_t PrepareInsertLargeGenerationsEnabled(
+    CommonFields& common, const PolicyFunctions& policy, size_t hash,
+    Group::NonIterableBitMaskType mask_empty, FindInfo target_group,
+    absl::FunctionRef<size_t(size_t)> recompute_hash) {
+  // NOLINTNEXTLINE(misc-static-assert)
+  ABSL_SWISSTABLE_ASSERT(SwisstableGenerationsEnabled());
+  if (common.should_rehash_for_bug_detection_on_insert()) {
+    // Move to a different heap allocation in order to detect bugs.
+    const size_t cap = common.capacity();
+    ResizeAllocatedTableWithSeedChange(
+        common, policy, common.growth_left() > 0 ? cap : NextCapacity(cap));
+    hash = recompute_hash(common.seed().seed());
+    std::tie(target_group, mask_empty) =
+        find_first_non_full_group(common, hash);
+  }
+  return PrepareInsertLargeImpl(common, policy, hash, mask_empty, target_group);
 }
 
 namespace {
@@ -1808,35 +2067,44 @@ template size_t TryFindNewIndexWithoutProbing(size_t h1, size_t old_index,
 // We need to instantiate ALL possible template combinations because we define
 // the function in the cc file.
 template size_t GrowSooTableToNextCapacityAndPrepareInsert<0, false>(
-    CommonFields&, const PolicyFunctions&, size_t, ctrl_t);
+    CommonFields&, const PolicyFunctions&, absl::FunctionRef<size_t(size_t)>,
+    bool);
 template size_t GrowSooTableToNextCapacityAndPrepareInsert<
-    OptimalMemcpySizeForSooSlotTransfer(1), true>(CommonFields&,
-                                                  const PolicyFunctions&,
-                                                  size_t, ctrl_t);
+    OptimalMemcpySizeForSooSlotTransfer(1), true>(
+    CommonFields&, const PolicyFunctions&, absl::FunctionRef<size_t(size_t)>,
+    bool);
 
 static_assert(VerifyOptimalMemcpySizeForSooSlotTransferRange(2, 3));
 template size_t GrowSooTableToNextCapacityAndPrepareInsert<
-    OptimalMemcpySizeForSooSlotTransfer(3), true>(CommonFields&,
-                                                  const PolicyFunctions&,
-                                                  size_t, ctrl_t);
+    OptimalMemcpySizeForSooSlotTransfer(3), true>(
+    CommonFields&, const PolicyFunctions&, absl::FunctionRef<size_t(size_t)>,
+    bool);
 
 static_assert(VerifyOptimalMemcpySizeForSooSlotTransferRange(4, 8));
 template size_t GrowSooTableToNextCapacityAndPrepareInsert<
-    OptimalMemcpySizeForSooSlotTransfer(8), true>(CommonFields&,
-                                                  const PolicyFunctions&,
-                                                  size_t, ctrl_t);
+    OptimalMemcpySizeForSooSlotTransfer(8), true>(
+    CommonFields&, const PolicyFunctions&, absl::FunctionRef<size_t(size_t)>,
+    bool);
 
 #if UINTPTR_MAX == UINT32_MAX
 static_assert(MaxSooSlotSize() == 8);
 #else
 static_assert(VerifyOptimalMemcpySizeForSooSlotTransferRange(9, 16));
 template size_t GrowSooTableToNextCapacityAndPrepareInsert<
-    OptimalMemcpySizeForSooSlotTransfer(16), true>(CommonFields&,
-                                                   const PolicyFunctions&,
-                                                   size_t, ctrl_t);
+    OptimalMemcpySizeForSooSlotTransfer(16), true>(
+    CommonFields&, const PolicyFunctions&, absl::FunctionRef<size_t(size_t)>,
+    bool);
 static_assert(MaxSooSlotSize() == 16);
 #endif
 
+template void* AllocateBackingArray<BackingArrayAlignment(alignof(size_t)),
+                                    std::allocator<char>>(void* alloc,
+                                                          size_t n);
+template void DeallocateBackingArray<BackingArrayAlignment(alignof(size_t)),
+                                     std::allocator<char>>(
+    void* alloc, size_t capacity, ctrl_t* ctrl, size_t slot_size,
+    size_t slot_align, bool had_infoz);
+
 }  // namespace container_internal
 ABSL_NAMESPACE_END
 }  // namespace absl