grpc 1.71.0 → 1.73.0

data/third_party/abseil-cpp/absl/container/internal/common_policy_traits.h

@@ -28,6 +28,15 @@ namespace absl {
 ABSL_NAMESPACE_BEGIN
 namespace container_internal {
 
+template <class Policy, class = void>
+struct policy_trait_element_is_owner : std::false_type {};
+
+template <class Policy>
+struct policy_trait_element_is_owner<
+    Policy,
+    std::enable_if_t<!std::is_void<typename Policy::element_is_owner>::value>>
+    : Policy::element_is_owner {};
+
 // Defines how slots are initialized/destroyed/moved.
 template <class Policy, class = void>
 struct common_policy_traits {

data/third_party/abseil-cpp/absl/container/internal/container_memory.h

@@ -17,6 +17,7 @@
 
 #include <cassert>
 #include <cstddef>
+#include <cstdint>
 #include <cstring>
 #include <memory>
 #include <new>

data/third_party/abseil-cpp/absl/container/internal/inlined_vector.h

@@ -83,8 +83,10 @@ using IsMoveAssignOk = std::is_move_assignable<ValueType<A>>;
 template <typename A>
 using IsSwapOk = absl::type_traits_internal::IsSwappable<ValueType<A>>;
 
-template <typename A, bool IsTriviallyDestructible =
-                          absl::is_trivially_destructible<ValueType<A>>::value>
+template <typename A,
+          bool IsTriviallyDestructible =
+              absl::is_trivially_destructible<ValueType<A>>::value &&
+              std::is_same<A, std::allocator<ValueType<A>>>::value>
 struct DestroyAdapter;
 
 template <typename A>

data/third_party/abseil-cpp/absl/container/internal/layout.h

@@ -306,7 +306,7 @@ constexpr size_t Max(size_t a, size_t b, Ts... rest) {
 template <class T>
 std::string TypeName() {
   std::string out;
-#if ABSL_INTERNAL_HAS_RTTI
+#ifdef ABSL_INTERNAL_HAS_RTTI
   absl::StrAppend(&out, "<",
                   absl::debugging_internal::DemangleString(typeid(T).name()),
                   ">");

data/third_party/abseil-cpp/absl/container/internal/raw_hash_set.cc

@@ -24,10 +24,12 @@
 #include "absl/base/config.h"
 #include "absl/base/dynamic_annotations.h"
 #include "absl/base/internal/endian.h"
+#include "absl/base/internal/raw_logging.h"
 #include "absl/base/optimization.h"
 #include "absl/container/internal/container_memory.h"
 #include "absl/container/internal/hashtablez_sampler.h"
 #include "absl/hash/hash.h"
+#include "absl/numeric/bits.h"
 
 namespace absl {
 ABSL_NAMESPACE_BEGIN
@@ -96,6 +98,16 @@ bool ShouldRehashForBugDetection(const ctrl_t* ctrl, size_t capacity) {
          RehashProbabilityConstant();
 }
 
+// Find a non-deterministic hash for single group table.
+// Last two bits are used to find a position for a newly inserted element after
+// resize.
+// This function is mixing all bits of hash and control pointer to maximize
+// entropy.
+size_t SingleGroupTableH1(size_t hash, ctrl_t* control) {
+  return static_cast<size_t>(absl::popcount(
+      hash ^ static_cast<size_t>(reinterpret_cast<uintptr_t>(control))));
+}
+
 }  // namespace
 
 GenerationType* EmptyGeneration() {
@@ -135,7 +147,7 @@ size_t PrepareInsertAfterSoo(size_t hash, size_t slot_size,
   // index 1 occupied, so we need to insert either at index 0 or index 2.
   assert(HashSetResizeHelper::SooSlotIndex() == 1);
   PrepareInsertCommon(common);
-  const size_t offset = H1(hash, common.control()) & 2;
+  const size_t offset = SingleGroupTableH1(hash, common.control()) & 2;
   common.growth_info().OverwriteEmptyAsFull();
   SetCtrlInSingleGroupTable(common, offset, H2(hash), slot_size);
   common.infoz().RecordInsert(hash, /*distance_from_desired=*/0);
@@ -338,7 +350,7 @@ void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy,
     c.infoz().RecordClearedReservation();
     c.infoz().RecordStorageChanged(0, soo_enabled ? SooCapacity() : 0);
     (*policy.dealloc)(c, policy);
-    c = soo_enabled ? CommonFields{soo_tag_t{}} : CommonFields{};
+    c = soo_enabled ? CommonFields{soo_tag_t{}} : CommonFields{non_soo_tag_t{}};
   }
 }
 
@@ -346,124 +358,124 @@ void HashSetResizeHelper::GrowIntoSingleGroupShuffleControlBytes(
     ctrl_t* __restrict new_ctrl, size_t new_capacity) const {
   assert(is_single_group(new_capacity));
   constexpr size_t kHalfWidth = Group::kWidth / 2;
-  constexpr size_t kQuarterWidth = Group::kWidth / 4;
-  assert(old_capacity_ < kHalfWidth);
-  static_assert(sizeof(uint64_t) >= kHalfWidth,
-                "Group size is too large. The ctrl bytes for half a group must "
-                "fit into a uint64_t for this implementation.");
-  static_assert(sizeof(uint64_t) <= Group::kWidth,
-                "Group size is too small. The ctrl bytes for a group must "
-                "cover a uint64_t for this implementation.");
-
-  const size_t half_old_capacity = old_capacity_ / 2;
-
-  // NOTE: operations are done with compile time known size = kHalfWidth.
+  ABSL_ASSUME(old_capacity_ < kHalfWidth);
+  ABSL_ASSUME(old_capacity_ > 0);
+  static_assert(Group::kWidth == 8 || Group::kWidth == 16,
+                "Group size is not supported.");
+
+  // NOTE: operations are done with compile time known size = 8.
   // Compiler optimizes that into single ASM operation.
 
-  // Load the bytes from half_old_capacity + 1. This contains the last half of
-  // old_ctrl bytes, followed by the sentinel byte, and then the first half of
-  // the cloned bytes. This effectively shuffles the control bytes.
-  uint64_t copied_bytes = 0;
-  copied_bytes =
-      absl::little_endian::Load64(old_ctrl() + half_old_capacity + 1);
+  // Load the bytes from old_capacity. This contains
+  // - the sentinel byte
+  // - all the old control bytes
+  // - the rest is filled with kEmpty bytes
+  // Example:
+  // old_ctrl =     012S012EEEEEEEEE...
+  // copied_bytes = S012EEEE
+  uint64_t copied_bytes =
+      absl::little_endian::Load64(old_ctrl() + old_capacity_);
 
   // We change the sentinel byte to kEmpty before storing to both the start of
   // the new_ctrl, and past the end of the new_ctrl later for the new cloned
   // bytes. Note that this is faster than setting the sentinel byte to kEmpty
   // after the copy directly in new_ctrl because we are limited on store
   // bandwidth.
-  constexpr uint64_t kEmptyXorSentinel =
+  static constexpr uint64_t kEmptyXorSentinel =
       static_cast<uint8_t>(ctrl_t::kEmpty) ^
       static_cast<uint8_t>(ctrl_t::kSentinel);
-  const uint64_t mask_convert_old_sentinel_to_empty =
-      kEmptyXorSentinel << (half_old_capacity * 8);
-  copied_bytes ^= mask_convert_old_sentinel_to_empty;
-
-  // Copy second half of bytes to the beginning. This correctly sets the bytes
-  // [0, old_capacity]. We potentially copy more bytes in order to have compile
-  // time known size. Mirrored bytes from the old_ctrl() will also be copied. In
-  // case of old_capacity_ == 3, we will copy 1st element twice.
-  // Examples:
-  // (old capacity = 1)
-  // old_ctrl = 0S0EEEEEEE...
-  // new_ctrl = E0EEEEEE??...
-  //
-  // (old capacity = 3)
-  // old_ctrl = 012S012EEEEE...
-  // new_ctrl = 12E012EE????...
-  //
-  // (old capacity = 7)
-  // old_ctrl = 0123456S0123456EE...
-  // new_ctrl = 456E0123?????????...
-  absl::little_endian::Store64(new_ctrl, copied_bytes);
 
-  // Set the space [old_capacity + 1, new_capacity] to empty as these bytes will
-  // not be written again. This is safe because
-  // NumControlBytes = new_capacity + kWidth and new_capacity >=
-  // old_capacity+1.
-  // Examples:
-  // (old_capacity = 3, new_capacity = 15)
-  // new_ctrl  = 12E012EE?????????????...??
-  // *new_ctrl = 12E0EEEEEEEEEEEEEEEE?...??
-  // position        /          S
-  //
-  // (old_capacity = 7, new_capacity = 15)
-  // new_ctrl  = 456E0123?????????????????...??
-  // *new_ctrl = 456E0123EEEEEEEEEEEEEEEE?...??
-  // position            /      S
-  std::memset(new_ctrl + old_capacity_ + 1, static_cast<int8_t>(ctrl_t::kEmpty),
-              Group::kWidth);
-
-  // Set the last kHalfWidth bytes to empty, to ensure the bytes all the way to
-  // the end are initialized.
-  // Examples:
-  // new_ctrl  = 12E0EEEEEEEEEEEEEEEE?...???????
-  // *new_ctrl = 12E0EEEEEEEEEEEEEEEE???EEEEEEEE
-  // position                   S       /
-  //
-  // new_ctrl  = 456E0123EEEEEEEEEEEEEEEE???????
-  // *new_ctrl = 456E0123EEEEEEEEEEEEEEEEEEEEEEE
-  // position                   S       /
-  std::memset(new_ctrl + NumControlBytes(new_capacity) - kHalfWidth,
-              static_cast<int8_t>(ctrl_t::kEmpty), kHalfWidth);
+  // Replace the first byte kSentinel with kEmpty.
+  // Resulting bytes will be shifted by one byte old control blocks.
+  // Example:
+  // old_ctrl = 012S012EEEEEEEEE...
+  // before =   S012EEEE
+  // after  =   E012EEEE
+  copied_bytes ^= kEmptyXorSentinel;
+
+  if (Group::kWidth == 8) {
+    // With group size 8, we can grow with two write operations.
+    assert(old_capacity_ < 8 && "old_capacity_ is too large for group size 8");
+    absl::little_endian::Store64(new_ctrl, copied_bytes);
+
+    static constexpr uint64_t kSentinal64 =
+        static_cast<uint8_t>(ctrl_t::kSentinel);
+
+    // Prepend kSentinel byte to the beginning of copied_bytes.
+    // We have maximum 3 non-empty bytes at the beginning of copied_bytes for
+    // group size 8.
+    // Example:
+    // old_ctrl = 012S012EEEE
+    // before =   E012EEEE
+    // after  =   SE012EEE
+    copied_bytes = (copied_bytes << 8) ^ kSentinal64;
+    absl::little_endian::Store64(new_ctrl + new_capacity, copied_bytes);
+    // Example for capacity 3:
+    // old_ctrl = 012S012EEEE
+    // After the first store:
+    //           >!
+    // new_ctrl = E012EEEE???????
+    // After the second store:
+    //                  >!
+    // new_ctrl = E012EEESE012EEE
+    return;
+  }
 
-  // Copy the first bytes to the end (starting at new_capacity +1) to set the
-  // cloned bytes. Note that we use the already copied bytes from old_ctrl here
-  // rather than copying from new_ctrl to avoid a Read-after-Write hazard, since
-  // new_ctrl was just written to. The first old_capacity-1 bytes are set
-  // correctly. Then there may be up to old_capacity bytes that need to be
-  // overwritten, and any remaining bytes will be correctly set to empty. This
-  // sets [new_capacity + 1, new_capacity +1 + old_capacity] correctly.
-  // Examples:
-  // new_ctrl  = 12E0EEEEEEEEEEEEEEEE?...???????
-  // *new_ctrl = 12E0EEEEEEEEEEEE12E012EEEEEEEEE
-  // position                   S/
-  //
-  // new_ctrl  = 456E0123EEEEEEEE?...???EEEEEEEE
-  // *new_ctrl = 456E0123EEEEEEEE456E0123EEEEEEE
-  // position                   S/
-  absl::little_endian::Store64(new_ctrl + new_capacity + 1, copied_bytes);
+  assert(Group::kWidth == 16);
 
-  // Set The remaining bytes at the end past the cloned bytes to empty. The
-  // incorrectly set bytes are [new_capacity + old_capacity + 2,
-  // min(new_capacity + 1 + kHalfWidth, new_capacity + old_capacity + 2 +
-  // half_old_capacity)]. Taking the difference, we need to set min(kHalfWidth -
-  // (old_capacity + 1), half_old_capacity)]. Since old_capacity < kHalfWidth,
-  // half_old_capacity < kQuarterWidth, so we set kQuarterWidth beginning at
-  // new_capacity + old_capacity + 2 to kEmpty.
-  // Examples:
-  // new_ctrl  = 12E0EEEEEEEEEEEE12E012EEEEEEEEE
-  // *new_ctrl = 12E0EEEEEEEEEEEE12E0EEEEEEEEEEE
-  // position                   S    /
-  //
-  // new_ctrl  = 456E0123EEEEEEEE456E0123EEEEEEE
-  // *new_ctrl = 456E0123EEEEEEEE456E0123EEEEEEE (no change)
-  // position                   S        /
-  std::memset(new_ctrl + new_capacity + old_capacity_ + 2,
-              static_cast<int8_t>(ctrl_t::kEmpty), kQuarterWidth);
-
-  // Finally, we set the new sentinel byte.
+  // Fill the second half of the main control bytes with kEmpty.
+  // For small capacity that may write into mirrored control bytes.
+  // It is fine as we will overwrite all the bytes later.
+  std::memset(new_ctrl + kHalfWidth, static_cast<int8_t>(ctrl_t::kEmpty),
+              kHalfWidth);
+  // Fill the second half of the mirrored control bytes with kEmpty.
+  std::memset(new_ctrl + new_capacity + kHalfWidth,
+              static_cast<int8_t>(ctrl_t::kEmpty), kHalfWidth);
+  // Copy the first half of the non-mirrored control bytes.
+  absl::little_endian::Store64(new_ctrl, copied_bytes);
   new_ctrl[new_capacity] = ctrl_t::kSentinel;
+  // Copy the first half of the mirrored control bytes.
+  absl::little_endian::Store64(new_ctrl + new_capacity + 1, copied_bytes);
+
+  // Example for growth capacity 1->3:
+  // old_ctrl =                  0S0EEEEEEEEEEEEEE
+  // new_ctrl at the end =       E0ESE0EEEEEEEEEEEEE
+  //                                    >!
+  // new_ctrl after 1st memset = ????????EEEEEEEE???
+  //                                       >!
+  // new_ctrl after 2nd memset = ????????EEEEEEEEEEE
+  //                            >!
+  // new_ctrl after 1st store =  E0EEEEEEEEEEEEEEEEE
+  // new_ctrl after kSentinel =  E0ESEEEEEEEEEEEEEEE
+  //                                >!
+  // new_ctrl after 2nd store =  E0ESE0EEEEEEEEEEEEE
+
+  // Example for growth capacity 3->7:
+  // old_ctrl =                  012S012EEEEEEEEEEEE
+  // new_ctrl at the end =       E012EEESE012EEEEEEEEEEE
+  //                                    >!
+  // new_ctrl after 1st memset = ????????EEEEEEEE???????
+  //                                           >!
+  // new_ctrl after 2nd memset = ????????EEEEEEEEEEEEEEE
+  //                            >!
+  // new_ctrl after 1st store =  E012EEEEEEEEEEEEEEEEEEE
+  // new_ctrl after kSentinel =  E012EEESEEEEEEEEEEEEEEE
+  //                                >!
+  // new_ctrl after 2nd store =  E012EEESE012EEEEEEEEEEE
+
+
+  // Example for growth capacity 7->15:
+  // old_ctrl =                  0123456S0123456EEEEEEEE
+  // new_ctrl at the end =       E0123456EEEEEEESE0123456EEEEEEE
+  //                                    >!
+  // new_ctrl after 1st memset = ????????EEEEEEEE???????????????
+  //                                                   >!
+  // new_ctrl after 2nd memset = ????????EEEEEEEE???????EEEEEEEE
+  //                            >!
+  // new_ctrl after 1st store =  E0123456EEEEEEEE???????EEEEEEEE
+  // new_ctrl after kSentinel =  E0123456EEEEEEES???????EEEEEEEE
+  //                                            >!
+  // new_ctrl after 2nd store =  E0123456EEEEEEESE0123456EEEEEEE
 }
 
 void HashSetResizeHelper::InitControlBytesAfterSoo(ctrl_t* new_ctrl, ctrl_t h2,
@@ -480,15 +492,10 @@ void HashSetResizeHelper::InitControlBytesAfterSoo(ctrl_t* new_ctrl, ctrl_t h2,
 
 void HashSetResizeHelper::GrowIntoSingleGroupShuffleTransferableSlots(
     void* new_slots, size_t slot_size) const {
-  assert(old_capacity_ > 0);
-  const size_t half_old_capacity = old_capacity_ / 2;
-
+  ABSL_ASSUME(old_capacity_ > 0);
   SanitizerUnpoisonMemoryRegion(old_slots(), slot_size * old_capacity_);
-  std::memcpy(new_slots,
-              SlotAddress(old_slots(), half_old_capacity + 1, slot_size),
-              slot_size * half_old_capacity);
-  std::memcpy(SlotAddress(new_slots, half_old_capacity + 1, slot_size),
-              old_slots(), slot_size * (half_old_capacity + 1));
+  std::memcpy(SlotAddress(new_slots, 1, slot_size), old_slots(),
+              slot_size * old_capacity_);
 }
 
 void HashSetResizeHelper::GrowSizeIntoSingleGroupTransferable(
@@ -588,6 +595,23 @@ const void* GetHashRefForEmptyHasher(const CommonFields& common) {
   return &common;
 }
 
+FindInfo HashSetResizeHelper::FindFirstNonFullAfterResize(const CommonFields& c,
+                                                          size_t old_capacity,
+                                                          size_t hash) {
+  size_t new_capacity = c.capacity();
+  if (!IsGrowingIntoSingleGroupApplicable(old_capacity, new_capacity)) {
+    return find_first_non_full(c, hash);
+  }
+
+  // We put the new element either at the beginning or at the end of the table
+  // with approximately equal probability.
+  size_t offset =
+      SingleGroupTableH1(hash, c.control()) & 1 ? 0 : new_capacity - 1;
+
+  assert(IsEmpty(c.control()[offset]));
+  return FindInfo{offset, 0};
+}
+
 size_t PrepareInsertNonSoo(CommonFields& common, size_t hash, FindInfo target,
                            const PolicyFunctions& policy) {
   // When there are no deleted slots in the table
@@ -638,6 +662,10 @@ size_t PrepareInsertNonSoo(CommonFields& common, size_t hash, FindInfo target,
   return target.offset;
 }
 
+void HashTableSizeOverflow() {
+  ABSL_RAW_LOG(FATAL, "Hash table size overflow");
+}
+
 }  // namespace container_internal
 ABSL_NAMESPACE_END
 }  // namespace absl