grpc 1.52.0 → 1.53.0

data/third_party/abseil-cpp/absl/crc/internal/crc_internal.h

@@ -0,0 +1,179 @@
+// Copyright 2022 The Abseil Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef ABSL_CRC_INTERNAL_CRC_INTERNAL_H_
+#define ABSL_CRC_INTERNAL_CRC_INTERNAL_H_
+
+#include <cstdint>
+#include <memory>
+#include <vector>
+
+#include "absl/base/internal/raw_logging.h"
+#include "absl/crc/internal/crc.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+
+namespace crc_internal {
+
+// Prefetch constants used in some Extend() implementations
+constexpr int kPrefetchHorizon = ABSL_CACHELINE_SIZE * 4;  // Prefetch this far
+// Shorter prefetch distance for smaller buffers
+constexpr int kPrefetchHorizonMedium = ABSL_CACHELINE_SIZE * 1;
+static_assert(kPrefetchHorizon >= 64, "CRCPrefetchHorizon less than loop len");
+
+// We require the Scramble() function:
+//  - to be reversible (Unscramble() must exist)
+//  - to be non-linear in the polynomial's Galois field (so the CRC of a
+//    scrambled CRC is not linearly affected by the scrambled CRC, even if
+//    using the same polynomial)
+//  - not to be its own inverse.  Preferably, if X=Scramble^N(X) and N!=0, then
+//    N is large.
+//  - to be fast.
+//  - not to change once defined.
+// We introduce non-linearity in two ways:
+//     Addition of a constant.
+//         - The carries introduce non-linearity; we use bits of an irrational
+//           (phi) to make it unlikely that we introduce no carries.
+//     Rotate by a constant number of bits.
+//         - We use floor(degree/2)+1, which does not divide the degree, and
+//           splits the bits nearly evenly, which makes it less likely the
+//           halves will be the same or one will be all zeroes.
+// We do both things to improve the chances of non-linearity in the face of
+// bit patterns with low numbers of bits set, while still being fast.
+// Below is the constant that we add.  The bits are the first 128 bits of the
+// fractional part of phi, with a 1 ored into the bottom bit to maximize the
+// cycle length of repeated adds.
+constexpr uint64_t kScrambleHi = (static_cast<uint64_t>(0x4f1bbcdcU) << 32) |
+                                 static_cast<uint64_t>(0xbfa53e0aU);
+constexpr uint64_t kScrambleLo = (static_cast<uint64_t>(0xf9ce6030U) << 32) |
+                                 static_cast<uint64_t>(0x2e76e41bU);
+
+class CRCImpl : public CRC {  // Implemention of the abstract class CRC
+ public:
+  using Uint32By256 = uint32_t[256];
+
+  CRCImpl() {}
+  ~CRCImpl() override = default;
+
+  // The internal version of CRC::New().
+  static CRCImpl* NewInternal();
+
+  void Empty(uint32_t* crc) const override;
+
+  // Fill in a table for updating a CRC by one word of 'word_size' bytes
+  // [last_lo, last_hi] contains the answer if the last bit in the word
+  // is set.
+  static void FillWordTable(uint32_t poly, uint32_t last, int word_size,
+                            Uint32By256* t);
+
+  // Build the table for extending by zeroes, returning the number of entries.
+  // For a in {1, 2, ..., ZEROES_BASE-1}, b in {0, 1, 2, 3, ...},
+  // entry j=a-1+(ZEROES_BASE-1)*b
+  // contains a polynomial Pi such that multiplying
+  // a CRC by Pi mod P, where P is the CRC polynomial, is equivalent to
+  // appending a*2**(ZEROES_BASE_LG*b) zero bytes to the original string.
+  static int FillZeroesTable(uint32_t poly, Uint32By256* t);
+
+  virtual void InitTables() = 0;
+
+ private:
+  CRCImpl(const CRCImpl&) = delete;
+  CRCImpl& operator=(const CRCImpl&) = delete;
+};
+
+// This is the 32-bit implementation.  It handles all sizes from 8 to 32.
+class CRC32 : public CRCImpl {
+ public:
+  CRC32() {}
+  ~CRC32() override {}
+
+  void Extend(uint32_t* crc, const void* bytes, size_t length) const override;
+  void ExtendByZeroes(uint32_t* crc, size_t length) const override;
+  void Scramble(uint32_t* crc) const override;
+  void Unscramble(uint32_t* crc) const override;
+  void UnextendByZeroes(uint32_t* crc, size_t length) const override;
+
+  void InitTables() override;
+
+ private:
+  // Common implementation guts for ExtendByZeroes and UnextendByZeroes().
+  //
+  // zeroes_table is a table as returned by FillZeroesTable(), containing
+  // polynomials representing CRCs of strings-of-zeros of various lenghts,
+  // and which can be combined by polynomial multiplication.  poly_table is
+  // a table of CRC byte extension values.  These tables are determined by
+  // the generator polynomial.
+  //
+  // These will be set to reverse_zeroes_ and reverse_table0_ for Unextend, and
+  // CRC32::zeroes_ and CRC32::table0_ for Extend.
+  void ExtendByZeroesImpl(uint32_t* crc, size_t length,
+                          const uint32_t zeroes_table[256],
+                          const uint32_t poly_table[256]) const;
+
+  uint32_t table0_[256];  // table of byte extensions
+  uint32_t zeroes_[256];  // table of zero extensions
+
+  // table of 4-byte extensions shifted by 12 bytes of zeroes
+  uint32_t table_[4][256];
+
+  // Reverse lookup tables, using the alternate polynomial used by
+  // UnextendByZeroes().
+  uint32_t reverse_table0_[256];  // table of reverse byte extensions
+  uint32_t reverse_zeroes_[256];  // table of reverse zero extensions
+
+  CRC32(const CRC32&) = delete;
+  CRC32& operator=(const CRC32&) = delete;
+};
+
+// Helpers
+
+// Return a bit mask containing len 1-bits.
+// Requires 0 < len <= sizeof(T)
+template <typename T>
+T MaskOfLength(int len) {
+  // shift 2 by len-1 rather than 1 by len because shifts of wordsize
+  // are undefined.
+  return (T(2) << (len - 1)) - 1;
+}
+
+// Rotate low-order "width" bits of "in" right by "r" bits,
+// setting other bits in word to arbitrary values.
+template <typename T>
+T RotateRight(T in, int width, int r) {
+  return (in << (width - r)) | ((in >> r) & MaskOfLength<T>(width - r));
+}
+
+// RoundUp<N>(p) returns the lowest address >= p aligned to an N-byte
+// boundary.  Requires that N is a power of 2.
+template <int alignment>
+const uint8_t* RoundUp(const uint8_t* p) {
+  static_assert((alignment & (alignment - 1)) == 0, "alignment is not 2^n");
+  constexpr uintptr_t mask = alignment - 1;
+  const uintptr_t as_uintptr = reinterpret_cast<uintptr_t>(p);
+  return reinterpret_cast<const uint8_t*>((as_uintptr + mask) & ~mask);
+}
+
+// Return a newly created CRC32AcceleratedX86ARMCombined if we can use Intel's
+// or ARM's CRC acceleration for a given polynomial.  Return nullptr otherwise.
+CRCImpl* TryNewCRC32AcceleratedX86ARMCombined();
+
+// Return all possible hardware accelerated implementations. For testing only.
+std::vector<std::unique_ptr<CRCImpl>> NewCRC32AcceleratedX86ARMCombinedAll();
+
+}  // namespace crc_internal
+ABSL_NAMESPACE_END
+}  // namespace absl
+
+#endif  // ABSL_CRC_INTERNAL_CRC_INTERNAL_H_

data/third_party/abseil-cpp/absl/crc/internal/crc_memcpy.h

@@ -0,0 +1,119 @@
+// Copyright 2022 The Abseil Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef ABSL_CRC_INTERNAL_CRC_MEMCPY_H_
+#define ABSL_CRC_INTERNAL_CRC_MEMCPY_H_
+
+#include <cstddef>
+#include <memory>
+
+#include "absl/base/config.h"
+#include "absl/crc/crc32c.h"
+
+// Defined if the class AcceleratedCrcMemcpyEngine exists.
+#if defined(__x86_64__) && defined(__SSE4_2__)
+#define ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE 1
+#elif defined(_MSC_VER) && defined(__AVX__)
+#define ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE 1
+#endif
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace crc_internal {
+
+class CrcMemcpyEngine {
+ public:
+  virtual ~CrcMemcpyEngine() = default;
+
+  virtual crc32c_t Compute(void* __restrict dst, const void* __restrict src,
+                           std::size_t length, crc32c_t initial_crc) const = 0;
+
+ protected:
+  CrcMemcpyEngine() = default;
+};
+
+class CrcMemcpy {
+ public:
+  static crc32c_t CrcAndCopy(void* __restrict dst, const void* __restrict src,
+                             std::size_t length,
+                             crc32c_t initial_crc = crc32c_t{0},
+                             bool non_temporal = false) {
+    static const ArchSpecificEngines engines = GetArchSpecificEngines();
+    auto* engine = non_temporal ? engines.non_temporal : engines.temporal;
+    return engine->Compute(dst, src, length, initial_crc);
+  }
+
+  // For testing only: get an architecture-specific engine for tests.
+  static std::unique_ptr<CrcMemcpyEngine> GetTestEngine(int vector,
+                                                        int integer);
+
+ private:
+  struct ArchSpecificEngines {
+    CrcMemcpyEngine* temporal;
+    CrcMemcpyEngine* non_temporal;
+  };
+
+  static ArchSpecificEngines GetArchSpecificEngines();
+};
+
+// Fallback CRC-memcpy engine.
+class FallbackCrcMemcpyEngine : public CrcMemcpyEngine {
+ public:
+  FallbackCrcMemcpyEngine() = default;
+  FallbackCrcMemcpyEngine(const FallbackCrcMemcpyEngine&) = delete;
+  FallbackCrcMemcpyEngine operator=(const FallbackCrcMemcpyEngine&) = delete;
+
+  crc32c_t Compute(void* __restrict dst, const void* __restrict src,
+                   std::size_t length, crc32c_t initial_crc) const override;
+};
+
+// CRC Non-Temporal-Memcpy engine.
+class CrcNonTemporalMemcpyEngine : public CrcMemcpyEngine {
+ public:
+  CrcNonTemporalMemcpyEngine() = default;
+  CrcNonTemporalMemcpyEngine(const CrcNonTemporalMemcpyEngine&) = delete;
+  CrcNonTemporalMemcpyEngine operator=(const CrcNonTemporalMemcpyEngine&) =
+      delete;
+
+  crc32c_t Compute(void* __restrict dst, const void* __restrict src,
+                   std::size_t length, crc32c_t initial_crc) const override;
+};
+
+// CRC Non-Temporal-Memcpy AVX engine.
+class CrcNonTemporalMemcpyAVXEngine : public CrcMemcpyEngine {
+ public:
+  CrcNonTemporalMemcpyAVXEngine() = default;
+  CrcNonTemporalMemcpyAVXEngine(const CrcNonTemporalMemcpyAVXEngine&) = delete;
+  CrcNonTemporalMemcpyAVXEngine operator=(
+      const CrcNonTemporalMemcpyAVXEngine&) = delete;
+
+  crc32c_t Compute(void* __restrict dst, const void* __restrict src,
+                   std::size_t length, crc32c_t initial_crc) const override;
+};
+
+// Copy source to destination and return the CRC32C of the data copied.  If an
+// accelerated version is available, use the accelerated version, otherwise use
+// the generic fallback version.
+inline crc32c_t Crc32CAndCopy(void* __restrict dst, const void* __restrict src,
+                              std::size_t length,
+                              crc32c_t initial_crc = crc32c_t{0},
+                              bool non_temporal = false) {
+  return CrcMemcpy::CrcAndCopy(dst, src, length, initial_crc, non_temporal);
+}
+
+}  // namespace crc_internal
+ABSL_NAMESPACE_END
+}  // namespace absl
+
+#endif  // ABSL_CRC_INTERNAL_CRC_MEMCPY_H_

data/third_party/abseil-cpp/absl/crc/internal/crc_memcpy_fallback.cc

@@ -0,0 +1,75 @@
+// Copyright 2022 The Abseil Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <cstdint>
+#include <memory>
+
+#include "absl/base/config.h"
+#include "absl/crc/crc32c.h"
+#include "absl/crc/internal/crc_memcpy.h"
+
+namespace absl {
+ABSL_NAMESPACE_BEGIN
+namespace crc_internal {
+
+absl::crc32c_t FallbackCrcMemcpyEngine::Compute(void* __restrict dst,
+                                                const void* __restrict src,
+                                                std::size_t length,
+                                                crc32c_t initial_crc) const {
+  constexpr size_t kBlockSize = 8192;
+  absl::crc32c_t crc = initial_crc;
+
+  const char* src_bytes = reinterpret_cast<const char*>(src);
+  char* dst_bytes = reinterpret_cast<char*>(dst);
+
+  // Copy + CRC loop - run 8k chunks until we are out of full chunks.  CRC
+  // then copy was found to be slightly more efficient in our test cases.
+  std::size_t offset = 0;
+  for (; offset + kBlockSize < length; offset += kBlockSize) {
+    crc = absl::ExtendCrc32c(crc,
+                             absl::string_view(src_bytes + offset, kBlockSize));
+    memcpy(dst_bytes + offset, src_bytes + offset, kBlockSize);
+  }
+
+  // Save some work if length is 0.
+  if (offset < length) {
+    std::size_t final_copy_size = length - offset;
+    crc = absl::ExtendCrc32c(
+        crc, absl::string_view(src_bytes + offset, final_copy_size));
+    memcpy(dst_bytes + offset, src_bytes + offset, final_copy_size);
+  }
+
+  return crc;
+}
+
+// Compile the following only if we don't have
+#ifndef ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE
+
+CrcMemcpy::ArchSpecificEngines CrcMemcpy::GetArchSpecificEngines() {
+  CrcMemcpy::ArchSpecificEngines engines;
+  engines.temporal = new FallbackCrcMemcpyEngine();
+  engines.non_temporal = new FallbackCrcMemcpyEngine();
+  return engines;
+}
+
+std::unique_ptr<CrcMemcpyEngine> CrcMemcpy::GetTestEngine(int /*vector*/,
+                                                          int /*integer*/) {
+  return std::make_unique<FallbackCrcMemcpyEngine>();
+}
+
+#endif  // ABSL_INTERNAL_HAVE_X86_64_ACCELERATED_CRC_MEMCPY_ENGINE
+
+}  // namespace crc_internal
+ABSL_NAMESPACE_END
+}  // namespace absl