grpc 1.84.1 → 1.85.0

data/third_party/abseil-cpp/absl/hash/internal/city.cc

@@ -21,13 +21,13 @@
 
 #include "absl/hash/internal/city.h"
 
-#include <string.h>  // for memcpy and memset
 #include <algorithm>
 
 #include "absl/base/config.h"
 #include "absl/base/internal/endian.h"
 #include "absl/base/internal/unaligned_access.h"
 #include "absl/base/optimization.h"
+#include "absl/numeric/bits.h"
 
 namespace absl {
 ABSL_NAMESPACE_BEGIN
@@ -68,11 +68,6 @@ static uint32_t fmix(uint32_t h) {
   return h;
 }
 
-static uint32_t Rotate32(uint32_t val, int shift) {
-  // Avoid shifting by 32: doing so yields an undefined result.
-  return shift == 0 ? val : ((val >> shift) | (val << (32 - shift)));
-}
-
 #undef PERMUTE3
 #define PERMUTE3(a, b, c) \
   do {                    \
@@ -83,10 +78,10 @@ static uint32_t Rotate32(uint32_t val, int shift) {
 static uint32_t Mur(uint32_t a, uint32_t h) {
   // Helper from Murmur3 for combining two 32-bit values.
   a *= c1;
-  a = Rotate32(a, 17);
+  a = absl::rotr(a, 17);
   a *= c2;
   h ^= a;
-  h = Rotate32(h, 19);
+  h = absl::rotr(h, 19);
   return h * 5 + 0xe6546b64;
 }
 
@@ -131,44 +126,44 @@ uint32_t CityHash32(const char *s, size_t len) {
   // len > 24
   uint32_t h = static_cast<uint32_t>(len), g = c1 * h, f = g;
 
-  uint32_t a0 = Rotate32(Fetch32(s + len - 4) * c1, 17) * c2;
-  uint32_t a1 = Rotate32(Fetch32(s + len - 8) * c1, 17) * c2;
-  uint32_t a2 = Rotate32(Fetch32(s + len - 16) * c1, 17) * c2;
-  uint32_t a3 = Rotate32(Fetch32(s + len - 12) * c1, 17) * c2;
-  uint32_t a4 = Rotate32(Fetch32(s + len - 20) * c1, 17) * c2;
+  uint32_t a0 = absl::rotr(Fetch32(s + len - 4) * c1, 17) * c2;
+  uint32_t a1 = absl::rotr(Fetch32(s + len - 8) * c1, 17) * c2;
+  uint32_t a2 = absl::rotr(Fetch32(s + len - 16) * c1, 17) * c2;
+  uint32_t a3 = absl::rotr(Fetch32(s + len - 12) * c1, 17) * c2;
+  uint32_t a4 = absl::rotr(Fetch32(s + len - 20) * c1, 17) * c2;
   h ^= a0;
-  h = Rotate32(h, 19);
+  h = absl::rotr(h, 19);
   h = h * 5 + 0xe6546b64;
   h ^= a2;
-  h = Rotate32(h, 19);
+  h = absl::rotr(h, 19);
   h = h * 5 + 0xe6546b64;
   g ^= a1;
-  g = Rotate32(g, 19);
+  g = absl::rotr(g, 19);
   g = g * 5 + 0xe6546b64;
   g ^= a3;
-  g = Rotate32(g, 19);
+  g = absl::rotr(g, 19);
   g = g * 5 + 0xe6546b64;
   f += a4;
-  f = Rotate32(f, 19);
+  f = absl::rotr(f, 19);
   f = f * 5 + 0xe6546b64;
   size_t iters = (len - 1) / 20;
   do {
-    uint32_t b0 = Rotate32(Fetch32(s) * c1, 17) * c2;
+    uint32_t b0 = absl::rotr(Fetch32(s) * c1, 17) * c2;
     uint32_t b1 = Fetch32(s + 4);
-    uint32_t b2 = Rotate32(Fetch32(s + 8) * c1, 17) * c2;
-    uint32_t b3 = Rotate32(Fetch32(s + 12) * c1, 17) * c2;
+    uint32_t b2 = absl::rotr(Fetch32(s + 8) * c1, 17) * c2;
+    uint32_t b3 = absl::rotr(Fetch32(s + 12) * c1, 17) * c2;
     uint32_t b4 = Fetch32(s + 16);
     h ^= b0;
-    h = Rotate32(h, 18);
+    h = absl::rotr(h, 18);
     h = h * 5 + 0xe6546b64;
     f += b1;
-    f = Rotate32(f, 19);
+    f = absl::rotr(f, 19);
     f = f * c1;
     g += b2;
-    g = Rotate32(g, 18);
+    g = absl::rotr(g, 18);
     g = g * 5 + 0xe6546b64;
     h ^= b3 + b1;
-    h = Rotate32(h, 19);
+    h = absl::rotr(h, 19);
     h = h * 5 + 0xe6546b64;
     g ^= b4;
     g = absl::gbswap_32(g) * 5;
@@ -178,26 +173,19 @@ uint32_t CityHash32(const char *s, size_t len) {
     PERMUTE3(f, h, g);
     s += 20;
   } while (--iters != 0);
-  g = Rotate32(g, 11) * c1;
-  g = Rotate32(g, 17) * c1;
-  f = Rotate32(f, 11) * c1;
-  f = Rotate32(f, 17) * c1;
-  h = Rotate32(h + g, 19);
+  g = absl::rotr(g, 11) * c1;
+  g = absl::rotr(g, 17) * c1;
+  f = absl::rotr(f, 11) * c1;
+  f = absl::rotr(f, 17) * c1;
+  h = absl::rotr(h + g, 19);
   h = h * 5 + 0xe6546b64;
-  h = Rotate32(h, 17) * c1;
-  h = Rotate32(h + f, 19);
+  h = absl::rotr(h, 17) * c1;
+  h = absl::rotr(h + f, 19);
   h = h * 5 + 0xe6546b64;
-  h = Rotate32(h, 17) * c1;
+  h = absl::rotr(h, 17) * c1;
   return h;
 }
 
-// Bitwise right rotate.  Normally this will compile to a single
-// instruction, especially if the shift is a manifest constant.
-static uint64_t Rotate(uint64_t val, int shift) {
-  // Avoid shifting by 64: doing so yields an undefined result.
-  return shift == 0 ? val : ((val >> shift) | (val << (64 - shift)));
-}
-
 static uint64_t ShiftMix(uint64_t val) { return val ^ (val >> 47); }
 
 static uint64_t HashLen16(uint64_t u, uint64_t v, uint64_t mul) {
@@ -220,8 +208,8 @@ static uint64_t HashLen0to16(const char *s, size_t len) {
     uint64_t mul = k2 + len * 2;
     uint64_t a = Fetch64(s) + k2;
     uint64_t b = Fetch64(s + len - 8);
-    uint64_t c = Rotate(b, 37) * mul + a;
-    uint64_t d = (Rotate(a, 25) + b) * mul;
+    uint64_t c = absl::rotr(b, 37) * mul + a;
+    uint64_t d = (absl::rotr(a, 25) + b) * mul;
     return HashLen16(c, d, mul);
   }
   if (len >= 4) {
@@ -248,8 +236,8 @@ static uint64_t HashLen17to32(const char *s, size_t len) {
   uint64_t b = Fetch64(s + 8);
   uint64_t c = Fetch64(s + len - 8) * mul;
   uint64_t d = Fetch64(s + len - 16) * k2;
-  return HashLen16(Rotate(a + b, 43) + Rotate(c, 30) + d,
-                   a + Rotate(b + k2, 18) + c, mul);
+  return HashLen16(absl::rotr(a + b, 43) + absl::rotr(c, 30) + d,
+                   a + absl::rotr(b + k2, 18) + c, mul);
 }
 
 // Return a 16-byte hash for 48 bytes.  Quick and dirty.
@@ -257,11 +245,11 @@ static uint64_t HashLen17to32(const char *s, size_t len) {
 static std::pair<uint64_t, uint64_t> WeakHashLen32WithSeeds(
     uint64_t w, uint64_t x, uint64_t y, uint64_t z, uint64_t a, uint64_t b) {
   a += w;
-  b = Rotate(b + a + z, 21);
+  b = absl::rotr(b + a + z, 21);
   uint64_t c = a;
   a += x;
   a += y;
-  b += Rotate(a, 44);
+  b += absl::rotr(a, 44);
   return std::make_pair(a + z, b + c);
 }
 
@@ -284,10 +272,10 @@ static uint64_t HashLen33to64(const char *s, size_t len) {
   uint64_t f = Fetch64(s + 24) * 9;
   uint64_t g = Fetch64(s + len - 8);
   uint64_t h = Fetch64(s + len - 16) * mul;
-  uint64_t u = Rotate(a + g, 43) + (Rotate(b, 30) + c) * 9;
+  uint64_t u = absl::rotr(a + g, 43) + (absl::rotr(b, 30) + c) * 9;
   uint64_t v = ((a + g) ^ d) + f + 1;
   uint64_t w = absl::gbswap_64((u + v) * mul) + h;
-  uint64_t x = Rotate(e + f, 42) + c;
+  uint64_t x = absl::rotr(e + f, 42) + c;
   uint64_t y = (absl::gbswap_64((v + w) * mul) + g) * mul;
   uint64_t z = e + f + c;
   a = absl::gbswap_64((x + z) * mul + y) + b;
@@ -320,11 +308,11 @@ uint64_t CityHash64(const char *s, size_t len) {
   // Decrease len to the nearest multiple of 64, and operate on 64-byte chunks.
   len = (len - 1) & ~static_cast<size_t>(63);
   do {
-    x = Rotate(x + y + v.first + Fetch64(s + 8), 37) * k1;
-    y = Rotate(y + v.second + Fetch64(s + 48), 42) * k1;
+    x = absl::rotr(x + y + v.first + Fetch64(s + 8), 37) * k1;
+    y = absl::rotr(y + v.second + Fetch64(s + 48), 42) * k1;
     x ^= w.second;
     y += v.first + Fetch64(s + 40);
-    z = Rotate(z + w.first, 33) * k1;
+    z = absl::rotr(z + w.first, 33) * k1;
     v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
     w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16));
     std::swap(z, x);

data/third_party/abseil-cpp/absl/hash/internal/hash.cc

@@ -14,52 +14,417 @@
 
 #include "absl/hash/internal/hash.h"
 
+#include <cassert>
 #include <cstddef>
 #include <cstdint>
 #include <type_traits>
 
 #include "absl/base/attributes.h"
 #include "absl/base/config.h"
-#include "absl/hash/internal/low_level_hash.h"
+#include "absl/base/internal/unaligned_access.h"
+#include "absl/base/optimization.h"
+#include "absl/base/prefetch.h"
+#include "absl/hash/internal/city.h"
+
+#ifdef ABSL_AES_INTERNAL_HAVE_X86_SIMD
+#error ABSL_AES_INTERNAL_HAVE_X86_SIMD cannot be directly set
+#elif defined(__SSE4_2__) && defined(__AES__)
+#define ABSL_AES_INTERNAL_HAVE_X86_SIMD
+#endif
+
+
+#ifdef ABSL_AES_INTERNAL_HAVE_X86_SIMD
+#include <smmintrin.h>
+#include <wmmintrin.h>
+#include <xmmintrin.h>
+#endif  // ABSL_AES_INTERNAL_HAVE_X86_SIMD
+
+#ifdef ABSL_AES_INTERNAL_HAVE_ARM_SIMD
+#error ABSL_AES_INTERNAL_HAVE_ARM_SIMD cannot be directly set
+#elif defined(ABSL_INTERNAL_HAVE_ARM_NEON) && defined(__ARM_FEATURE_CRYPTO)
+#include <arm_neon.h>
+#define ABSL_AES_INTERNAL_HAVE_ARM_SIMD
+#endif  // ABSL_INTERNAL_HAVE_ARM_NEON
 
 namespace absl {
 ABSL_NAMESPACE_BEGIN
 namespace hash_internal {
 
-uint64_t MixingHashState::CombineLargeContiguousImpl32(
-    uint64_t state, const unsigned char* first, size_t len) {
+namespace {
+
+void PrefetchFutureDataToLocalCache(const uint8_t* ptr) {
+  PrefetchToLocalCache(ptr + 5 * ABSL_CACHELINE_SIZE);
+}
+
+#if defined(ABSL_AES_INTERNAL_HAVE_X86_SIMD) || \
+    defined(ABSL_AES_INTERNAL_HAVE_ARM_SIMD)
+
+#if defined(ABSL_AES_INTERNAL_HAVE_X86_SIMD)
+using Vector128 = __m128i;
+
+inline Vector128 Load128(const uint8_t* ptr) {
+  return _mm_loadu_si128(reinterpret_cast<const Vector128*>(ptr));
+}
+
+inline Vector128 Set128(uint64_t a, uint64_t b) {
+  return _mm_set_epi64x(static_cast<int64_t>(a), static_cast<int64_t>(b));
+}
+
+inline Vector128 Add128(Vector128 a, Vector128 b) {
+  return _mm_add_epi64(a, b);
+}
+
+inline Vector128 Sub128(Vector128 a, Vector128 b) {
+  return _mm_sub_epi64(a, b);
+}
+
+// Bits of the second argument to Encrypt128/Decrypt128 are XORed with the
+// first argument after encryption/decryption.
+
+inline Vector128 Encrypt128(Vector128 data, Vector128 key) {
+  return _mm_aesenc_si128(data, key);
+}
+
+inline Vector128 Decrypt128(Vector128 data, Vector128 key) {
+  return _mm_aesdec_si128(data, key);
+}
+
+// We use each value as the first argument to shuffle all the bits around. We do
+// not add any salt to the state or loaded data, instead we vary instructions
+// used to mix bits Encrypt128/Decrypt128 and Add128/Sub128. On x86,
+// Add128/Sub128 are combined to one instruction with data loading like
+// `vpaddq xmm1, xmm0, xmmword ptr [rdi]`.
+
+inline Vector128 MixA(Vector128 a, Vector128 state) {
+  return Decrypt128(Add128(state, a), state);
+}
+
+inline Vector128 MixB(Vector128 b, Vector128 state) {
+  return Decrypt128(Sub128(state, b), state);
+}
+
+inline Vector128 MixC(Vector128 c, Vector128 state) {
+  return Encrypt128(Add128(state, c), state);
+}
+
+inline Vector128 MixD(Vector128 d, Vector128 state) {
+  return Encrypt128(Sub128(state, d), state);
+}
+
+inline uint64_t ExtractLow64(Vector128 v) {
+  return static_cast<uint64_t>(_mm_cvtsi128_si64(v));
+}
+
+inline uint64_t ExtractHigh64(Vector128 v) {
+  return static_cast<uint64_t>(_mm_extract_epi64(v, 1));
+}
+
+inline uint64_t Mix4x16Vectors(Vector128 a, Vector128 b, Vector128 c,
+                               Vector128 d) {
+  Vector128 res128 =
+      Add128(Encrypt128(Add128(a, c), d), Decrypt128(Sub128(b, d), a));
+  uint64_t x64 = ExtractLow64(res128);
+  uint64_t y64 = ExtractHigh64(res128);
+  return x64 ^ y64;
+}
+
+#else  // ABSL_AES_INTERNAL_HAVE_ARM_SIMD
+
+using Vector128 = uint8x16_t;
+
+inline Vector128 Load128(const uint8_t* ptr) { return vld1q_u8(ptr); }
+
+inline Vector128 Set128(uint64_t a, uint64_t b) {
+  return vreinterpretq_u8_u64(vsetq_lane_u64(a, vdupq_n_u64(b), 1));
+}
+
+inline Vector128 Add128(Vector128 a, Vector128 b) {
+  return vreinterpretq_u8_u64(
+      vaddq_u64(vreinterpretq_u64_u8(a), vreinterpretq_u64_u8(b)));
+}
+
+// Bits of the second argument to Decrypt128/Encrypt128 are XORed with the
+// state argument BEFORE encryption (in x86 version they are XORed after).
+
+inline Vector128 Encrypt128(Vector128 data, Vector128 key) {
+  return vaesmcq_u8(vaeseq_u8(data, key));
+}
+
+inline Vector128 Decrypt128(Vector128 data, Vector128 key) {
+  return vaesimcq_u8(vaesdq_u8(data, key));
+}
+
+// We use decryption for a, b and encryption for c, d. That helps us to avoid
+// collisions for trivial byte rotations. Mix4x16Vectors later uses
+// encrypted/decrypted pairs differently to ensure that the order of blocks is
+// important for the hash value.
+// We also avoid using Add128/Sub128 instructions because state is being mixed
+// before encryption/decryption. On ARM, there is no fusion of load and add/sub
+// instructions so it is more expensive to use them.
+
+inline Vector128 MixA(Vector128 a, Vector128 state) {
+  return Decrypt128(a, state);
+}
+
+inline Vector128 MixB(Vector128 b, Vector128 state) {
+  return Decrypt128(b, state);
+}
+
+inline Vector128 MixC(Vector128 c, Vector128 state) {
+  return Encrypt128(c, state);
+}
+
+inline Vector128 MixD(Vector128 d, Vector128 state) {
+  return Encrypt128(d, state);
+}
+
+inline uint64_t ExtractLow64(Vector128 v) {
+  return vgetq_lane_u64(vreinterpretq_u64_u8(v), 0);
+}
+
+inline uint64_t ExtractHigh64(Vector128 v) {
+  return vgetq_lane_u64(vreinterpretq_u64_u8(v), 1);
+}
+
+uint64_t Mix4x16Vectors(Vector128 a, Vector128 b, Vector128 c, Vector128 d) {
+  Vector128 res128 = Add128(Encrypt128(a, c), Decrypt128(b, d));
+  uint64_t x64 = ExtractLow64(res128);
+  uint64_t y64 = ExtractHigh64(res128);
+  return x64 ^ y64;
+}
+
+#endif  // ABSL_AES_INTERNAL_HAVE_X86_SIMD
+
+uint64_t LowLevelHash33To64(uint64_t seed, const uint8_t* ptr, size_t len) {
+  assert(len > 32);
+  assert(len <= 64);
+  Vector128 state = Set128(seed, len);
+  Vector128 a = Load128(ptr);
+  Vector128 b = Load128(ptr + 16);
+  auto* last32_ptr = ptr + len - 32;
+  Vector128 c = Load128(last32_ptr);
+  Vector128 d = Load128(last32_ptr + 16);
+
+  Vector128 na = MixA(a, state);
+  Vector128 nb = MixB(b, state);
+  Vector128 nc = MixC(c, state);
+  Vector128 nd = MixD(d, state);
+
+  // We perform another round of encryption to mix bits between two halves of
+  // the input.
+  return Mix4x16Vectors(na, nb, nc, nd);
+}
+
+[[maybe_unused]] ABSL_ATTRIBUTE_NOINLINE uint64_t
+LowLevelHashLenGt64(uint64_t seed, const void* data, size_t len) {
+  assert(len > 64);
+  const uint8_t* ptr = static_cast<const uint8_t*>(data);
+  const uint8_t* last_32_ptr = ptr + len - 32;
+
+  // If we have more than 64 bytes, we're going to handle chunks of 64
+  // bytes at a time. We're going to build up four separate hash states
+  // which we will then hash together. This avoids short dependency chains.
+  Vector128 state0 = Set128(seed, len);
+  Vector128 state1 = state0;
+  Vector128 state2 = state1;
+  Vector128 state3 = state2;
+
+  // Mixing two 128-bit vectors at a time with corresponding states.
+  // All variables are mixed slightly differently to avoid hash collision
+  // due to trivial byte rotation.
+  // We combine state and data with _mm_add_epi64/_mm_sub_epi64 before applying
+  // AES encryption to make hash function dependent on the order of the blocks.
+  // See comments in LowLevelHash33To64 for more considerations.
+  auto mix_ab = [&state0,
+                 &state1](const uint8_t* p) ABSL_ATTRIBUTE_ALWAYS_INLINE {
+    Vector128 a = Load128(p);
+    Vector128 b = Load128(p + 16);
+    state0 = MixA(a, state0);
+    state1 = MixB(b, state1);
+  };
+  auto mix_cd = [&state2,
+                 &state3](const uint8_t* p) ABSL_ATTRIBUTE_ALWAYS_INLINE {
+    Vector128 c = Load128(p);
+    Vector128 d = Load128(p + 16);
+    state2 = MixC(c, state2);
+    state3 = MixD(d, state3);
+  };
+
+  do {
+    PrefetchFutureDataToLocalCache(ptr);
+    mix_ab(ptr);
+    mix_cd(ptr + 32);
+
+    ptr += 64;
+    len -= 64;
+  } while (len > 64);
+
+  // We now have a data `ptr` with at most 64 bytes.
+  if (len > 32) {
+    mix_ab(ptr);
+  }
+  mix_cd(last_32_ptr);
+
+  return Mix4x16Vectors(state0, state1, state2, state3);
+}
+#else
+uint64_t Mix32Bytes(const uint8_t* ptr, uint64_t current_state) {
+  uint64_t a = absl::base_internal::UnalignedLoad64(ptr);
+  uint64_t b = absl::base_internal::UnalignedLoad64(ptr + 8);
+  uint64_t c = absl::base_internal::UnalignedLoad64(ptr + 16);
+  uint64_t d = absl::base_internal::UnalignedLoad64(ptr + 24);
+
+  uint64_t cs0 = Mix(a ^ kStaticRandomData[1], b ^ current_state);
+  uint64_t cs1 = Mix(c ^ kStaticRandomData[2], d ^ current_state);
+  return cs0 ^ cs1;
+}
+
+uint64_t LowLevelHash33To64(uint64_t seed, const uint8_t* ptr, size_t len) {
+  assert(len > 32);
+  assert(len <= 64);
+  uint64_t current_state = seed ^ kStaticRandomData[0] ^ len;
+  const uint8_t* last_32_ptr = ptr + len - 32;
+  return Mix32Bytes(last_32_ptr, Mix32Bytes(ptr, current_state));
+}
+
+[[maybe_unused]] ABSL_ATTRIBUTE_NOINLINE uint64_t
+LowLevelHashLenGt64(uint64_t seed, const void* data, size_t len) {
+  assert(len > 64);
+  const uint8_t* ptr = static_cast<const uint8_t*>(data);
+  uint64_t current_state = seed ^ kStaticRandomData[0] ^ len;
+  const uint8_t* last_32_ptr = ptr + len - 32;
+  // If we have more than 64 bytes, we're going to handle chunks of 64
+  // bytes at a time. We're going to build up four separate hash states
+  // which we will then hash together. This avoids short dependency chains.
+  uint64_t duplicated_state0 = current_state;
+  uint64_t duplicated_state1 = current_state;
+  uint64_t duplicated_state2 = current_state;
+
+  do {
+    PrefetchFutureDataToLocalCache(ptr);
+
+    uint64_t a = absl::base_internal::UnalignedLoad64(ptr);
+    uint64_t b = absl::base_internal::UnalignedLoad64(ptr + 8);
+    uint64_t c = absl::base_internal::UnalignedLoad64(ptr + 16);
+    uint64_t d = absl::base_internal::UnalignedLoad64(ptr + 24);
+    uint64_t e = absl::base_internal::UnalignedLoad64(ptr + 32);
+    uint64_t f = absl::base_internal::UnalignedLoad64(ptr + 40);
+    uint64_t g = absl::base_internal::UnalignedLoad64(ptr + 48);
+    uint64_t h = absl::base_internal::UnalignedLoad64(ptr + 56);
+
+    current_state = Mix(a ^ kStaticRandomData[1], b ^ current_state);
+    duplicated_state0 = Mix(c ^ kStaticRandomData[2], d ^ duplicated_state0);
+
+    duplicated_state1 = Mix(e ^ kStaticRandomData[3], f ^ duplicated_state1);
+    duplicated_state2 = Mix(g ^ kStaticRandomData[4], h ^ duplicated_state2);
+
+    ptr += 64;
+    len -= 64;
+  } while (len > 64);
+
+  current_state = (current_state ^ duplicated_state0) ^
+                  (duplicated_state1 + duplicated_state2);
+  // We now have a data `ptr` with at most 64 bytes and the current state
+  // of the hashing state machine stored in current_state.
+  if (len > 32) {
+    current_state = Mix32Bytes(ptr, current_state);
+  }
+
+  // We now have a data `ptr` with at most 32 bytes and the current state
+  // of the hashing state machine stored in current_state. But we can
+  // safely read from `ptr + len - 32`.
+  return Mix32Bytes(last_32_ptr, current_state);
+}
+#endif  // ABSL_AES_INTERNAL_HAVE_X86_SIMD
+
+[[maybe_unused]] uint64_t LowLevelHashLenGt32(uint64_t seed, const void* data,
+                                              size_t len) {
+  assert(len > 32);
+  if (ABSL_PREDICT_FALSE(len > 64)) {
+    return LowLevelHashLenGt64(seed, data, len);
+  }
+  return LowLevelHash33To64(seed, static_cast<const uint8_t*>(data), len);
+}
+
+ABSL_ATTRIBUTE_ALWAYS_INLINE inline uint64_t HashBlockOn32Bit(
+    uint64_t state, const unsigned char* data, size_t len) {
+  // TODO(b/417141985): expose and use CityHash32WithSeed.
+  // Note: we can't use PrecombineLengthMix here because len can be up to 1024.
+  return CombineRawImpl(
+      state + len,
+      hash_internal::CityHash32(reinterpret_cast<const char*>(data), len));
+}
+
+ABSL_ATTRIBUTE_NOINLINE uint64_t
+SplitAndCombineOn32Bit(uint64_t state, const unsigned char* first, size_t len) {
   while (len >= PiecewiseChunkSize()) {
-    state = Mix(
-        state ^ hash_internal::CityHash32(reinterpret_cast<const char*>(first),
-                                          PiecewiseChunkSize()),
-        kMul);
+    state = HashBlockOn32Bit(state, first, PiecewiseChunkSize());
     len -= PiecewiseChunkSize();
     first += PiecewiseChunkSize();
   }
+  // Do not call CombineContiguousImpl for empty range since it is modifying
+  // state.
+  if (len == 0) {
+    return state;
+  }
   // Handle the remainder.
   return CombineContiguousImpl(state, first, len,
                                std::integral_constant<int, 4>{});
 }
 
-uint64_t MixingHashState::CombineLargeContiguousImpl64(
-    uint64_t state, const unsigned char* first, size_t len) {
+ABSL_ATTRIBUTE_ALWAYS_INLINE inline uint64_t HashBlockOn64Bit(
+    uint64_t state, const unsigned char* data, size_t len) {
+#ifdef ABSL_HAVE_INTRINSIC_INT128
+  return LowLevelHashLenGt32(state, data, len);
+#else
+  return hash_internal::CityHash64WithSeed(reinterpret_cast<const char*>(data),
+                                           len, state);
+#endif
+}
+
+ABSL_ATTRIBUTE_NOINLINE uint64_t
+SplitAndCombineOn64Bit(uint64_t state, const unsigned char* first, size_t len) {
   while (len >= PiecewiseChunkSize()) {
-    state = Mix(state ^ Hash64(first, PiecewiseChunkSize()), kMul);
+    state = HashBlockOn64Bit(state, first, PiecewiseChunkSize());
     len -= PiecewiseChunkSize();
     first += PiecewiseChunkSize();
   }
+  // Do not call CombineContiguousImpl for empty range since it is modifying
+  // state.
+  if (len == 0) {
+    return state;
+  }
   // Handle the remainder.
   return CombineContiguousImpl(state, first, len,
                                std::integral_constant<int, 8>{});
 }
 
-ABSL_CONST_INIT const void* const MixingHashState::kSeed = &kSeed;
+}  // namespace
 
-uint64_t MixingHashState::LowLevelHashImpl(const unsigned char* data,
-                                           size_t len) {
-  return LowLevelHashLenGt32(data, len, Seed(), &kStaticRandomData[0]);
+uint64_t CombineLargeContiguousImplOn32BitLengthGt8(uint64_t state,
+                                                    const unsigned char* first,
+                                                    size_t len) {
+  assert(len > 8);
+  assert(sizeof(size_t) == 4);  // NOLINT(misc-static-assert)
+  if (ABSL_PREDICT_TRUE(len <= PiecewiseChunkSize())) {
+    return HashBlockOn32Bit(state, first, len);
+  }
+  return SplitAndCombineOn32Bit(state, first, len);
+}
+
+uint64_t CombineLargeContiguousImplOn64BitLengthGt32(uint64_t state,
+                                                     const unsigned char* first,
+                                                     size_t len) {
+  assert(len > 32);
+  assert(sizeof(size_t) == 8);  // NOLINT(misc-static-assert)
+  if (ABSL_PREDICT_TRUE(len <= PiecewiseChunkSize())) {
+    return HashBlockOn64Bit(state, first, len);
+  }
+  return SplitAndCombineOn64Bit(state, first, len);
 }
 
+ABSL_CONST_INIT const void* const MixingHashState::kSeed = &kSeed;
+
 }  // namespace hash_internal
 ABSL_NAMESPACE_END
 }  // namespace absl