grpc 1.81.1 → 1.85.0

data/third_party/abseil-cpp/absl/container/internal/container_memory.h

@@ -26,6 +26,7 @@
 #include <utility>
 
 #include "absl/base/config.h"
+#include "absl/hash/hash.h"
 #include "absl/memory/memory.h"
 #include "absl/meta/type_traits.h"
 #include "absl/utility/utility.h"
@@ -43,7 +44,12 @@ ABSL_NAMESPACE_BEGIN
 namespace container_internal {
 
 template <size_t Alignment>
-struct alignas(Alignment) AlignedType {};
+struct alignas(Alignment) AlignedType {
+  // When alignment is sufficient for the allocated memory to store pointers,
+  // include a pointer member so that swisstable backing arrays end up in the
+  // pointer-containing partition for heap partitioning.
+  std::conditional_t<(Alignment < alignof(void*)), char, void*> pointer;
+};
 
 // Allocates at least n bytes aligned to the specified alignment.
 // Alignment must be a power of 2. It must be positive.
@@ -57,8 +63,8 @@ void* Allocate(Alloc* alloc, size_t n) {
   static_assert(Alignment > 0, "");
   assert(n && "n must be positive");
   using M = AlignedType<Alignment>;
-  using A = typename absl::allocator_traits<Alloc>::template rebind_alloc<M>;
-  using AT = typename absl::allocator_traits<Alloc>::template rebind_traits<M>;
+  using A = typename std::allocator_traits<Alloc>::template rebind_alloc<M>;
+  using AT = typename std::allocator_traits<Alloc>::template rebind_traits<M>;
   // On macOS, "mem_alloc" is a #define with one argument defined in
   // rpc/types.h, so we can't name the variable "mem_alloc" and initialize it
   // with the "foo(bar)" syntax.
@@ -75,7 +81,7 @@ template <class Allocator, class ValueType>
 constexpr auto IsDestructionTrivial() {
   constexpr bool result =
       std::is_trivially_destructible<ValueType>::value &&
-      std::is_same<typename absl::allocator_traits<
+      std::is_same<typename std::allocator_traits<
                        Allocator>::template rebind_alloc<char>,
                    std::allocator<char>>::value;
   return std::integral_constant<bool, result>();
@@ -88,8 +94,8 @@ void Deallocate(Alloc* alloc, void* p, size_t n) {
   static_assert(Alignment > 0, "");
   assert(n && "n must be positive");
   using M = AlignedType<Alignment>;
-  using A = typename absl::allocator_traits<Alloc>::template rebind_alloc<M>;
-  using AT = typename absl::allocator_traits<Alloc>::template rebind_traits<M>;
+  using A = typename std::allocator_traits<Alloc>::template rebind_alloc<M>;
+  using AT = typename std::allocator_traits<Alloc>::template rebind_traits<M>;
   // On macOS, "mem_alloc" is a #define with one argument defined in
   // rpc/types.h, so we can't name the variable "mem_alloc" and initialize it
   // with the "foo(bar)" syntax.
@@ -104,8 +110,8 @@ namespace memory_internal {
 // specified in the tuple.
 template <class Alloc, class T, class Tuple, size_t... I>
 void ConstructFromTupleImpl(Alloc* alloc, T* ptr, Tuple&& t,
-                            absl::index_sequence<I...>) {
-  absl::allocator_traits<Alloc>::construct(
+                            std::index_sequence<I...>) {
+  std::allocator_traits<Alloc>::construct(
       *alloc, ptr, std::get<I>(std::forward<Tuple>(t))...);
 }
 
@@ -121,14 +127,15 @@ struct WithConstructedImplF {
 
 template <class T, class Tuple, size_t... Is, class F>
 decltype(std::declval<F>()(std::declval<T>())) WithConstructedImpl(
-    Tuple&& t, absl::index_sequence<Is...>, F&& f) {
+    Tuple&& t, std::index_sequence<Is...>, F&& f) {
   return WithConstructedImplF<T, F>{std::forward<F>(f)}(
       std::get<Is>(std::forward<Tuple>(t))...);
 }
 
 template <class T, size_t... Is>
-auto TupleRefImpl(T&& t, absl::index_sequence<Is...>)
+auto TupleRefImpl(T&& t, std::index_sequence<Is...>)
     -> decltype(std::forward_as_tuple(std::get<Is>(std::forward<T>(t))...)) {
+  // NOLINTNEXTLINE(bugprone-use-after-move)
   return std::forward_as_tuple(std::get<Is>(std::forward<T>(t))...);
 }
 
@@ -137,11 +144,11 @@ auto TupleRefImpl(T&& t, absl::index_sequence<Is...>)
 template <class T>
 auto TupleRef(T&& t) -> decltype(TupleRefImpl(
     std::forward<T>(t),
-    absl::make_index_sequence<
+    std::make_index_sequence<
         std::tuple_size<typename std::decay<T>::type>::value>())) {
   return TupleRefImpl(
       std::forward<T>(t),
-      absl::make_index_sequence<
+      std::make_index_sequence<
           std::tuple_size<typename std::decay<T>::type>::value>());
 }
 
@@ -162,7 +169,7 @@ template <class Alloc, class T, class Tuple>
 void ConstructFromTuple(Alloc* alloc, T* ptr, Tuple&& t) {
   memory_internal::ConstructFromTupleImpl(
       alloc, ptr, std::forward<Tuple>(t),
-      absl::make_index_sequence<
+      std::make_index_sequence<
           std::tuple_size<typename std::decay<Tuple>::type>::value>());
 }
 
@@ -173,7 +180,7 @@ decltype(std::declval<F>()(std::declval<T>())) WithConstructed(Tuple&& t,
                                                                F&& f) {
   return memory_internal::WithConstructedImpl<T>(
       std::forward<Tuple>(t),
-      absl::make_index_sequence<
+      std::make_index_sequence<
           std::tuple_size<typename std::decay<Tuple>::type>::value>(),
       std::forward<F>(f));
 }
@@ -343,11 +350,11 @@ union map_slot_type {
   ~map_slot_type() = delete;
   using value_type = std::pair<const K, V>;
   using mutable_value_type =
-      std::pair<absl::remove_const_t<K>, absl::remove_const_t<V>>;
+      std::pair<std::remove_const_t<K>, std::remove_const_t<V>>;
 
   value_type value;
   mutable_value_type mutable_value;
-  absl::remove_const_t<K> key;
+  std::remove_const_t<K> key;
 };
 
 template <class K, class V>
@@ -355,7 +362,7 @@ struct map_slot_policy {
   using slot_type = map_slot_type<K, V>;
   using value_type = std::pair<const K, V>;
   using mutable_value_type =
-      std::pair<absl::remove_const_t<K>, absl::remove_const_t<V>>;
+      std::pair<std::remove_const_t<K>, std::remove_const_t<V>>;
 
  private:
   static void emplace(slot_type* slot) {
@@ -390,11 +397,11 @@ struct map_slot_policy {
   static void construct(Allocator* alloc, slot_type* slot, Args&&... args) {
     emplace(slot);
     if (kMutableKeys::value) {
-      absl::allocator_traits<Allocator>::construct(*alloc, &slot->mutable_value,
-                                                   std::forward<Args>(args)...);
+      std::allocator_traits<Allocator>::construct(*alloc, &slot->mutable_value,
+                                                  std::forward<Args>(args)...);
     } else {
-      absl::allocator_traits<Allocator>::construct(*alloc, &slot->value,
-                                                   std::forward<Args>(args)...);
+      std::allocator_traits<Allocator>::construct(*alloc, &slot->value,
+                                                  std::forward<Args>(args)...);
     }
   }
 
@@ -403,11 +410,11 @@ struct map_slot_policy {
   static void construct(Allocator* alloc, slot_type* slot, slot_type* other) {
     emplace(slot);
     if (kMutableKeys::value) {
-      absl::allocator_traits<Allocator>::construct(
+      std::allocator_traits<Allocator>::construct(
           *alloc, &slot->mutable_value, std::move(other->mutable_value));
     } else {
-      absl::allocator_traits<Allocator>::construct(*alloc, &slot->value,
-                                                   std::move(other->value));
+      std::allocator_traits<Allocator>::construct(*alloc, &slot->value,
+                                                  std::move(other->value));
     }
   }
 
@@ -416,16 +423,16 @@ struct map_slot_policy {
   static void construct(Allocator* alloc, slot_type* slot,
                         const slot_type* other) {
     emplace(slot);
-    absl::allocator_traits<Allocator>::construct(*alloc, &slot->value,
-                                                 other->value);
+    std::allocator_traits<Allocator>::construct(*alloc, &slot->value,
+                                                other->value);
   }
 
   template <class Allocator>
   static auto destroy(Allocator* alloc, slot_type* slot) {
     if (kMutableKeys::value) {
-      absl::allocator_traits<Allocator>::destroy(*alloc, &slot->mutable_value);
+      std::allocator_traits<Allocator>::destroy(*alloc, &slot->mutable_value);
     } else {
-      absl::allocator_traits<Allocator>::destroy(*alloc, &slot->value);
+      std::allocator_traits<Allocator>::destroy(*alloc, &slot->value);
     }
     return IsDestructionTrivial<Allocator, value_type>();
   }
@@ -453,35 +460,108 @@ struct map_slot_policy {
     }
 
     if (kMutableKeys::value) {
-      absl::allocator_traits<Allocator>::construct(
+      std::allocator_traits<Allocator>::construct(
           *alloc, &new_slot->mutable_value, std::move(old_slot->mutable_value));
     } else {
-      absl::allocator_traits<Allocator>::construct(*alloc, &new_slot->value,
-                                                   std::move(old_slot->value));
+      std::allocator_traits<Allocator>::construct(*alloc, &new_slot->value,
+                                                  std::move(old_slot->value));
     }
     destroy(alloc, old_slot);
     return is_relocatable;
   }
 };
 
+// Suppress erroneous uninitialized memory errors on GCC. For example, GCC
+// thinks that the call to slot_array() in find_or_prepare_insert() is reading
+// uninitialized memory, but slot_array is only called there when the table is
+// non-empty and this memory is initialized when the table is non-empty.
+#if !defined(__clang__) && defined(__GNUC__)
+#define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(x)                    \
+  _Pragma("GCC diagnostic push")                                   \
+      _Pragma("GCC diagnostic ignored \"-Wmaybe-uninitialized\"")  \
+          _Pragma("GCC diagnostic ignored \"-Wuninitialized\"") x; \
+  _Pragma("GCC diagnostic pop")
+#define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(x) \
+  ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(return x)
+#else
+#define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED(x) x
+#define ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(x) return x
+#endif
+
+// Variadic arguments hash function that ignore the rest of the arguments.
+// Useful for usage with policy traits.
+template <class Hash, bool kIsDefault>
+struct HashElement {
+  HashElement(const Hash& h, size_t s) : hash(h), seed(s) {}
+
+  template <class K, class... Args>
+  size_t operator()(const K& key, Args&&...) const {
+    if constexpr (kIsDefault) {
+      // TODO(b/384509507): resolve `no header providing
+      // "absl::hash_internal::SupportsHashWithSeed" is directly included`.
+      // Maybe we should make "internal/hash.h" be a separate library.
+      return absl::hash_internal::HashWithSeed().hash(hash, key, seed);
+    }
+    // NOLINTNEXTLINE(clang-diagnostic-sign-conversion)
+    return hash(key) ^ seed;
+  }
+  const Hash& hash;
+  size_t seed;
+};
+
+// No arguments function hash function for a specific key.
+template <class Hash, class Key, bool kIsDefault>
+struct HashKey {
+  HashKey(const Hash& h, const Key& k) : hash(h), key(k) {}
+
+  size_t operator()(size_t seed) const {
+    return HashElement<Hash, kIsDefault>{hash, seed}(key);
+  }
+  const Hash& hash;
+  const Key& key;
+};
+
+// Variadic arguments equality function that ignore the rest of the arguments.
+// Useful for usage with policy traits.
+template <class K1, class KeyEqual>
+struct EqualElement {
+  template <class K2, class... Args>
+  bool operator()(const K2& lhs, Args&&...) const {
+    ABSL_SWISSTABLE_IGNORE_UNINITIALIZED_RETURN(eq(lhs, rhs));
+  }
+  const K1& rhs;
+  const KeyEqual& eq;
+};
+
 // Type erased function for computing hash of the slot.
-using HashSlotFn = size_t (*)(const void* hash_fn, void* slot);
+using HashSlotFn = size_t (*)(const void* hash_fn, void* slot, size_t seed);
 
 // Type erased function to apply `Fn` to data inside of the `slot`.
 // The data is expected to have type `T`.
-template <class Fn, class T>
-size_t TypeErasedApplyToSlotFn(const void* fn, void* slot) {
+template <class Fn, class T, bool kIsDefault>
+size_t TypeErasedApplyToSlotFn(const void* fn, void* slot, size_t seed) {
   const auto* f = static_cast<const Fn*>(fn);
-  return (*f)(*static_cast<const T*>(slot));
+  return HashElement<Fn, kIsDefault>{*f, seed}(*static_cast<const T*>(slot));
 }
 
 // Type erased function to apply `Fn` to data inside of the `*slot_ptr`.
 // The data is expected to have type `T`.
-template <class Fn, class T>
-size_t TypeErasedDerefAndApplyToSlotFn(const void* fn, void* slot_ptr) {
+template <class Fn, class T, bool kIsDefault>
+size_t TypeErasedDerefAndApplyToSlotFn(const void* fn, void* slot_ptr,
+                                       size_t seed) {
+  const auto* f = static_cast<const Fn*>(fn);
+  const T* slot = *static_cast<T**>(slot_ptr);
+  return HashElement<Fn, kIsDefault>{*f, seed}(*slot);
+}
+
+// Type erased function to apply `Fn` to data inside of the `slot_ptr->first`.
+// The data is expected to have type `T`.
+template <class Fn, class T, bool kIsDefault>
+size_t TypeErasedDerefAndApplyToSlotFirstFn(const void* fn, void* slot_ptr,
+                                            size_t seed) {
   const auto* f = static_cast<const Fn*>(fn);
-  const T* slot = *static_cast<const T**>(slot_ptr);
-  return (*f)(*slot);
+  const T* slot = *static_cast<T**>(slot_ptr);
+  return HashElement<Fn, kIsDefault>{*f, seed}(slot->first);
 }
 
 }  // namespace container_internal

data/third_party/abseil-cpp/absl/container/internal/hash_function_defaults.h

@@ -79,6 +79,18 @@ struct StringHash {
   size_t operator()(const absl::Cord& v) const {
     return absl::Hash<absl::Cord>{}(v);
   }
+
+ private:
+  friend struct absl::hash_internal::HashWithSeed;
+
+  size_t hash_with_seed(absl::string_view v, size_t seed) const {
+    return absl::hash_internal::HashWithSeed().hash(
+        absl::Hash<absl::string_view>{}, v, seed);
+  }
+  size_t hash_with_seed(const absl::Cord& v, size_t seed) const {
+    return absl::hash_internal::HashWithSeed().hash(absl::Hash<absl::Cord>{}, v,
+                                                    seed);
+  }
 };
 
 struct StringEq {
@@ -187,14 +199,14 @@ template <typename T, typename E = void>
 struct HasAbslContainerHash : std::false_type {};
 
 template <typename T>
-struct HasAbslContainerHash<T, absl::void_t<typename T::absl_container_hash>>
+struct HasAbslContainerHash<T, std::void_t<typename T::absl_container_hash>>
     : std::true_type {};
 
 template <typename T, typename E = void>
 struct HasAbslContainerEq : std::false_type {};
 
 template <typename T>
-struct HasAbslContainerEq<T, absl::void_t<typename T::absl_container_eq>>
+struct HasAbslContainerEq<T, std::void_t<typename T::absl_container_eq>>
     : std::true_type {};
 
 template <typename T, typename E = void>

data/third_party/abseil-cpp/absl/container/internal/hash_policy_traits.h

@@ -22,6 +22,7 @@
 #include <utility>
 
 #include "absl/container/internal/common_policy_traits.h"
+#include "absl/container/internal/container_memory.h"
 #include "absl/meta/type_traits.h"
 
 namespace absl {
@@ -30,14 +31,14 @@ namespace container_internal {
 
 // Defines how slots are initialized/destroyed/moved.
 template <class Policy, class = void>
-struct hash_policy_traits : common_policy_traits<Policy> {
+  struct hash_policy_traits : common_policy_traits<Policy> {
   // The type of the keys stored in the hashtable.
   using key_type = typename Policy::key_type;
 
  private:
   struct ReturnKey {
     template <class Key,
-              absl::enable_if_t<std::is_lvalue_reference<Key>::value, int> = 0>
+              std::enable_if_t<std::is_lvalue_reference<Key>::value, int> = 0>
     static key_type& Impl(Key&& k, int) {
       return *std::launder(
           const_cast<key_type*>(std::addressof(std::forward<Key>(k))));
@@ -62,7 +63,7 @@ struct hash_policy_traits : common_policy_traits<Policy> {
   struct ConstantIteratorsImpl : std::false_type {};
 
   template <class P>
-  struct ConstantIteratorsImpl<P, absl::void_t<typename P::constant_iterators>>
+  struct ConstantIteratorsImpl<P, std::void_t<typename P::constant_iterators>>
       : P::constant_iterators {};
 
  public:
@@ -145,9 +146,7 @@ struct hash_policy_traits : common_policy_traits<Policy> {
     return P::value(elem);
   }
 
-  using HashSlotFn = size_t (*)(const void* hash_fn, void* slot);
-
-  template <class Hash>
+  template <class Hash, bool kIsDefault>
   static constexpr HashSlotFn get_hash_slot_fn() {
 // get_hash_slot_fn may return nullptr to signal that non type erased function
 // should be used. GCC warns against comparing function address with nullptr.
@@ -156,9 +155,9 @@ struct hash_policy_traits : common_policy_traits<Policy> {
 // silent error: the address of * will never be NULL [-Werror=address]
 #pragma GCC diagnostic ignored "-Waddress"
 #endif
-    return Policy::template get_hash_slot_fn<Hash>() == nullptr
-               ? &hash_slot_fn_non_type_erased<Hash>
-               : Policy::template get_hash_slot_fn<Hash>();
+    return Policy::template get_hash_slot_fn<Hash, kIsDefault>() == nullptr
+               ? &hash_slot_fn_non_type_erased<Hash, kIsDefault>
+               : Policy::template get_hash_slot_fn<Hash, kIsDefault>();
 #if defined(__GNUC__) && !defined(__clang__)
 #pragma GCC diagnostic pop
 #endif
@@ -168,19 +167,12 @@ struct hash_policy_traits : common_policy_traits<Policy> {
   static constexpr bool soo_enabled() { return soo_enabled_impl(Rank1{}); }
 
  private:
-  template <class Hash>
-  struct HashElement {
-    template <class K, class... Args>
-    size_t operator()(const K& key, Args&&...) const {
-      return h(key);
-    }
-    const Hash& h;
-  };
-
-  template <class Hash>
-  static size_t hash_slot_fn_non_type_erased(const void* hash_fn, void* slot) {
-    return Policy::apply(HashElement<Hash>{*static_cast<const Hash*>(hash_fn)},
-                         Policy::element(static_cast<slot_type*>(slot)));
+  template <class Hash, bool kIsDefault>
+  static size_t hash_slot_fn_non_type_erased(const void* hash_fn, void* slot,
+                                             size_t seed) {
+    return Policy::apply(
+        HashElement<Hash, kIsDefault>{*static_cast<const Hash*>(hash_fn), seed},
+        Policy::element(static_cast<slot_type*>(slot)));
   }
 
   // Use go/ranked-overloads for dispatching. Rank1 is preferred.

data/third_party/abseil-cpp/absl/container/internal/hashtable_control_bytes.h

@@ -84,18 +84,13 @@ class NonIterableBitMask {
  public:
   explicit NonIterableBitMask(T mask) : mask_(mask) {}
 
-  explicit operator bool() const { return this->mask_ != 0; }
+  explicit operator bool() const { return mask_ != 0; }
 
   // Returns the index of the lowest *abstract* bit set in `self`.
   uint32_t LowestBitSet() const {
     return container_internal::TrailingZeros(mask_) >> Shift;
   }
 
-  // Returns the index of the highest *abstract* bit set in `self`.
-  uint32_t HighestBitSet() const {
-    return static_cast<uint32_t>((bit_width(mask_) - 1) >> Shift);
-  }
-
   // Returns the number of trailing zero *abstract* bits.
   uint32_t TrailingZeros() const {
     return container_internal::TrailingZeros(mask_) >> Shift;
@@ -217,6 +212,9 @@ static_assert(
 static_assert(ctrl_t::kDeleted == static_cast<ctrl_t>(-2),
               "ctrl_t::kDeleted must be -2 to make the implementation of "
               "ConvertSpecialToEmptyAndFullToDeleted efficient");
+static_assert(ctrl_t::kEmpty == static_cast<ctrl_t>(-128),
+              "ctrl_t::kEmpty must be -128 to use saturated subtraction in"
+              " ConvertSpecialToEmptyAndFullToDeleted");
 
 // Helpers for checking the state of a control byte.
 inline bool IsEmpty(ctrl_t c) { return c == ctrl_t::kEmpty; }
@@ -276,8 +274,12 @@ inline __m128i _mm_cmpgt_epi8_fixed(__m128i a, __m128i b) {
 
 struct GroupSse2Impl {
   static constexpr size_t kWidth = 16;  // the number of slots per group
-  using BitMaskType = BitMask<uint16_t, kWidth>;
-  using NonIterableBitMaskType = NonIterableBitMask<uint16_t, kWidth>;
+  // There are only 16 bits, but using uint32_t instead of uint16_t allows for
+  // better codegen. In particular, there is no blsr instruction for a 16 bit
+  // register, but there is for a 32 bit register (used in BitMask::operator++).
+  using MaskInt = uint32_t;
+  using BitMaskType = BitMask<MaskInt, kWidth>;
+  using NonIterableBitMaskType = NonIterableBitMask<MaskInt, kWidth>;
 
   explicit GroupSse2Impl(const ctrl_t* pos) {
     ctrl = _mm_loadu_si128(reinterpret_cast<const __m128i*>(pos));
@@ -286,64 +288,70 @@ struct GroupSse2Impl {
   // Returns a bitmask representing the positions of slots that match hash.
   BitMaskType Match(h2_t hash) const {
     auto match = _mm_set1_epi8(static_cast<char>(hash));
-    return BitMaskType(
-        static_cast<uint16_t>(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl))));
+    return BitMaskType(MoveMask(_mm_cmpeq_epi8(match, ctrl)));
   }
 
   // Returns a bitmask representing the positions of empty slots.
   NonIterableBitMaskType MaskEmpty() const {
 #ifdef ABSL_INTERNAL_HAVE_SSSE3
     // This only works because ctrl_t::kEmpty is -128.
-    return NonIterableBitMaskType(
-        static_cast<uint16_t>(_mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl))));
+    return NonIterableBitMaskType(MoveMask(_mm_sign_epi8(ctrl, ctrl)));
 #else
     auto match = _mm_set1_epi8(static_cast<char>(ctrl_t::kEmpty));
-    return NonIterableBitMaskType(
-        static_cast<uint16_t>(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl))));
+    return NonIterableBitMaskType(MoveMask(_mm_cmpeq_epi8(match, ctrl)));
 #endif
   }
 
   // Returns a bitmask representing the positions of full slots.
   // Note: for `is_small()` tables group may contain the "same" slot twice:
   // original and mirrored.
-  BitMaskType MaskFull() const {
-    return BitMaskType(static_cast<uint16_t>(_mm_movemask_epi8(ctrl) ^ 0xffff));
-  }
+  BitMaskType MaskFull() const { return BitMaskType(MoveMask(ctrl) ^ 0xffff); }
 
   // Returns a bitmask representing the positions of non full slots.
   // Note: this includes: kEmpty, kDeleted, kSentinel.
   // It is useful in contexts when kSentinel is not present.
-  auto MaskNonFull() const {
-    return BitMaskType(static_cast<uint16_t>(_mm_movemask_epi8(ctrl)));
-  }
+  auto MaskNonFull() const { return BitMaskType(MoveMask(ctrl)); }
 
   // Returns a bitmask representing the positions of empty or deleted slots.
   NonIterableBitMaskType MaskEmptyOrDeleted() const {
     auto special = _mm_set1_epi8(static_cast<char>(ctrl_t::kSentinel));
-    return NonIterableBitMaskType(static_cast<uint16_t>(
-        _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl))));
+    return NonIterableBitMaskType(
+        MoveMask(_mm_cmpgt_epi8_fixed(special, ctrl)));
   }
 
-  // Returns the number of trailing empty or deleted elements in the group.
-  uint32_t CountLeadingEmptyOrDeleted() const {
-    auto special = _mm_set1_epi8(static_cast<char>(ctrl_t::kSentinel));
-    return TrailingZeros(static_cast<uint32_t>(
-        _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1));
+  // Returns a bitmask representing the positions of full or sentinel slots.
+  // Note: for `is_small()` tables group may contain the "same" slot twice:
+  // original and mirrored.
+  NonIterableBitMaskType MaskFullOrSentinel() const {
+    auto special = _mm_set1_epi8(static_cast<char>(ctrl_t::kSentinel) - 1);
+    return NonIterableBitMaskType(
+        MoveMask(_mm_cmpgt_epi8_fixed(ctrl, special)));
   }
 
   void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const {
+    // Take advantage of the fact that kEmpty is already the smallest signed
+    // char value, and using a saturated subtraction will not affect it.
+    // All special values have the MSB set, so after an AND with MSBS, we
+    // are left with -128 for special values and 0 for full. After applying
+    // subs 2, we arrive at the result of -128(kEmpty) for special and
+    // -2(kDeleted) for full.
     auto msbs = _mm_set1_epi8(static_cast<char>(-128));
-    auto x126 = _mm_set1_epi8(126);
-#ifdef ABSL_INTERNAL_HAVE_SSSE3
-    auto res = _mm_or_si128(_mm_shuffle_epi8(x126, ctrl), msbs);
-#else
-    auto zero = _mm_setzero_si128();
-    auto special_mask = _mm_cmpgt_epi8_fixed(zero, ctrl);
-    auto res = _mm_or_si128(msbs, _mm_andnot_si128(special_mask, x126));
-#endif
+    auto twos = _mm_set1_epi8(static_cast<char>(2));
+    auto res = _mm_subs_epi8(_mm_and_si128(msbs, ctrl), twos);
     _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), res);
   }
 
+  static MaskInt MoveMask(__m128i xmm) {
+    auto mask = static_cast<MaskInt>(_mm_movemask_epi8(xmm));
+#ifdef __clang__
+    // TODO(b/472522597): Without the inline asm, clang ends up generating an
+    // unnecessary movzx to zero the upper bits of the output, but those bits
+    // are already zero. See https://godbolt.org/z/G6xW1Ecbx.
+    asm("" : "+r"(mask));  // NOLINT
+#endif
+    return mask;
+  }
+
   __m128i ctrl;
 };
 #endif  // ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2
@@ -406,17 +414,13 @@ struct GroupAArch64Impl {
     return NonIterableBitMaskType(mask);
   }
 
-  uint32_t CountLeadingEmptyOrDeleted() const {
-    uint64_t mask =
-        vget_lane_u64(vreinterpret_u64_u8(vcle_s8(
-                          vdup_n_s8(static_cast<int8_t>(ctrl_t::kSentinel)),
-                          vreinterpret_s8_u8(ctrl))),
-                      0);
-    // Similar to MaskEmptyorDeleted() but we invert the logic to invert the
-    // produced bitfield. We then count number of trailing zeros.
-    // Clang and GCC optimize countr_zero to rbit+clz without any check for 0,
-    // so we should be fine.
-    return static_cast<uint32_t>(countr_zero(mask)) >> 3;
+  NonIterableBitMaskType MaskFullOrSentinel() const {
+    uint64_t mask = vget_lane_u64(
+        vreinterpret_u64_u8(
+            vcgt_s8(vreinterpret_s8_u8(ctrl),
+                    vdup_n_s8(static_cast<int8_t>(ctrl_t::kSentinel) - 1))),
+        0);
+    return NonIterableBitMaskType(mask);
   }
 
   void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const {
@@ -481,12 +485,8 @@ struct GroupPortableImpl {
     return NonIterableBitMaskType((ctrl & ~(ctrl << 7)) & kMsbs8Bytes);
   }
 
-  uint32_t CountLeadingEmptyOrDeleted() const {
-    // ctrl | ~(ctrl >> 7) will have the lowest bit set to zero for kEmpty and
-    // kDeleted. We lower all other bits and count number of trailing zeros.
-    constexpr uint64_t bits = 0x0101010101010101ULL;
-    return static_cast<uint32_t>(countr_zero((ctrl | ~(ctrl >> 7)) & bits) >>
-                                 3);
+  auto MaskFullOrSentinel() const {
+    return NonIterableBitMaskType((~ctrl | (ctrl << 7)) & kMsbs8Bytes);
   }
 
   void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const {

data/third_party/abseil-cpp/absl/container/internal/hashtablez_sampler.cc

@@ -22,6 +22,7 @@
 #include <cstdint>
 #include <functional>
 #include <limits>
+#include <utility>
 
 #include "absl/base/attributes.h"
 #include "absl/base/config.h"
@@ -86,7 +87,6 @@ void HashtablezInfo::PrepareForSampling(int64_t stride,
   total_probe_length.store(0, std::memory_order_relaxed);
   hashes_bitwise_or.store(0, std::memory_order_relaxed);
   hashes_bitwise_and.store(~size_t{}, std::memory_order_relaxed);
-  hashes_bitwise_xor.store(0, std::memory_order_relaxed);
   max_reserve.store(0, std::memory_order_relaxed);
 
   create_time = absl::Now();
@@ -94,8 +94,9 @@ void HashtablezInfo::PrepareForSampling(int64_t stride,
   // The inliner makes hardcoded skip_count difficult (especially when combined
   // with LTO).  We use the ability to exclude stacks by regex when encoding
   // instead.
-  depth = absl::GetStackTrace(stack, HashtablezInfo::kMaxStackDepth,
-                              /* skip_count= */ 0);
+  depth = static_cast<unsigned>(
+      absl::GetStackTrace(stack, HashtablezInfo::kMaxStackDepth,
+                          /* skip_count= */ 0));
   inline_element_size = inline_element_size_value;
   key_size = key_size_value;
   value_size = value_size_value;
@@ -147,7 +148,7 @@ HashtablezInfo* SampleSlow(SamplingState& next_sample,
                            size_t value_size, uint16_t soo_capacity) {
   if (ABSL_PREDICT_FALSE(ShouldForceSampling())) {
     next_sample.next_sample = 1;
-    const int64_t old_stride = exchange(next_sample.sample_stride, 1);
+    const int64_t old_stride = std::exchange(next_sample.sample_stride, 1);
     HashtablezInfo* result = GlobalHashtablezSampler().Register(
         old_stride, inline_element_size, key_size, value_size, soo_capacity);
     return result;
@@ -229,8 +230,8 @@ void RecordStorageChangedSlow(HashtablezInfo* info, size_t size,
   }
 }
 
-void RecordInsertSlow(HashtablezInfo* info, size_t hash,
-                      size_t distance_from_desired) {
+void RecordInsertMissSlow(HashtablezInfo* info, size_t hash,
+                          size_t distance_from_desired) {
   // SwissTables probe in groups of 16, so scale this to count items probes and
   // not offset from desired.
   size_t probe_length = distance_from_desired;
@@ -242,7 +243,6 @@ void RecordInsertSlow(HashtablezInfo* info, size_t hash,
 
   info->hashes_bitwise_and.fetch_and(hash, std::memory_order_relaxed);
   info->hashes_bitwise_or.fetch_or(hash, std::memory_order_relaxed);
-  info->hashes_bitwise_xor.fetch_xor(hash, std::memory_order_relaxed);
   info->max_probe_length.store(
       std::max(info->max_probe_length.load(std::memory_order_relaxed),
                probe_length),