@nxtedition/rocksdb 15.4.0 → 15.5.0

package/deps/rocksdb/rocksdb/util/bit_fields.h

@@ -19,7 +19,7 @@ namespace ROCKSDB_NAMESPACE {
 // fields into atomic variables to reduce the need for locking in concurrent
 // code and/or to simplify reasoning on and accommodation of different
 // interesting, bug-prone interleavings. Convenient atomic wrappers
-// (RelaxedAtomic, AcqRelAtomic) are provided below to aid usage with atomics,
+// (RelaxedAtomic, Atomic) are provided below to aid usage with atomics,
 // especially for CAS updates, but it is even possible to combine operations on
 // multiple bit fields into a single non-CAS atomic operation using Transforms
 // below.
@@ -70,7 +70,7 @@ struct BitFields {
 
   // Return a copy with the given field modified
   template <typename BitFieldT>
-  Derived With(typename BitFieldT::V value) const {
+  constexpr Derived With(typename BitFieldT::V value) const {
     static_assert(std::is_same_v<typename BitFieldT::Parent, Derived>);
     Derived rv = static_cast<const Derived&>(*this);
     BitFieldT::SetIn(rv, value);
@@ -125,24 +125,26 @@ struct BitFields {
 // For building atomic updates affecting one or more fields, assuming all the
 // updates are bitwise-or.
 template <typename BitFieldsT>
-struct OrTransform {
+struct OrTransformer {
   using U = typename BitFieldsT::U;
   U to_or = 0;
   // + for general combine
-  OrTransform<BitFieldsT> operator+(OrTransform<BitFieldsT> other) const {
-    return OrTransform<BitFieldsT>{to_or | other.to_or};
+  OrTransformer<BitFieldsT> operator+(
+      const OrTransformer<BitFieldsT>& other) const {
+    return OrTransformer<BitFieldsT>{to_or | other.to_or};
   }
 };
 
 // For building atomic updates affecting one or more fields, assuming all the
 // updates are bitwise-and.
 template <typename BitFieldsT>
-struct AndTransform {
+struct AndTransformer {
   using U = typename BitFieldsT::U;
   U to_and = 0;
   // + for general combine
-  AndTransform<BitFieldsT> operator+(AndTransform<BitFieldsT> other) const {
-    return AndTransform<BitFieldsT>{to_and & other.to_and};
+  AndTransformer<BitFieldsT> operator+(
+      const AndTransformer<BitFieldsT>& other) const {
+    return AndTransformer<BitFieldsT>{to_and & other.to_and};
   }
 };
 
@@ -152,7 +154,7 @@ struct AndTransform {
 // corresponding preconditions. (NOTE that when representing a subtraction, we
 // rely on overflow of the unsigned representation.)
 template <typename BitFieldsT>
-struct AddTransform {
+struct AddTransformer {
   using U = typename BitFieldsT::U;
   U to_add = 0;
 #ifndef NDEBUG
@@ -162,7 +164,7 @@ struct AddTransform {
   };
   std::vector<Precondition> preconditions;
 #endif  // NDEBUG
-  void AssertPreconditions([[maybe_unused]] U from) {
+  void AssertPreconditions([[maybe_unused]] U from) const {
 #ifndef NDEBUG
     for (auto p : preconditions) {
       U tmp = (from & p.mask) + p.piece;
@@ -174,8 +176,9 @@ struct AddTransform {
 #endif  // NDEBUG
   }
   // + for general combine
-  AddTransform<BitFieldsT> operator+(AddTransform<BitFieldsT> other) const {
-    AddTransform<BitFieldsT> rv{to_add + other.to_add};
+  AddTransformer<BitFieldsT> operator+(
+      const AddTransformer<BitFieldsT>& other) const {
+    AddTransformer<BitFieldsT> rv{to_add + other.to_add};
 #ifndef NDEBUG
     rv.preconditions = preconditions;
     rv.preconditions.insert(rv.preconditions.end(), other.preconditions.begin(),
@@ -214,14 +217,18 @@ struct BoolBitField {
     return (bf.underlying & (U{1} << kBitOffset)) != 0;
   }
   static void SetIn(ParentBase& bf, bool value) {
+    // NOTE: avoiding conditional branches is usually best for speed on modern
+    // processors
     bf.underlying =
         (bf.underlying & ~(U{1} << kBitOffset)) | (U{value} << kBitOffset);
   }
-  static OrTransform<BitFieldsT> SetTransform() {
-    return OrTransform<BitFieldsT>{U{1} << kBitOffset};
+  static OrTransformer<BitFieldsT> SetTransform() { return Or(true); }
+  static OrTransformer<BitFieldsT> Or(bool b) {
+    return OrTransformer<BitFieldsT>{U{b} << kBitOffset};
   }
-  static AndTransform<BitFieldsT> ClearTransform() {
-    return AndTransform<BitFieldsT>{~(U{1} << kBitOffset)};
+  static AndTransformer<BitFieldsT> ClearTransform() { return And(false); }
+  static AndTransformer<BitFieldsT> And(bool b) {
+    return AndTransformer<BitFieldsT>{~(U{!b} << kBitOffset)};
   }
 };
 
@@ -258,18 +265,31 @@ struct UnsignedBitField {
     bf.underlying |= static_cast<U>(value & kMask) << kBitOffset;
   }
 
-  // Create a transfor for clearing this field to zero.
-  static AndTransform<BitFieldsT> ClearTransform() {
-    return AndTransform<BitFieldsT>{~(static_cast<U>(kMask) << kBitOffset)};
+  // Create a transform for clearing this field to zero.
+  static AndTransformer<BitFieldsT> ClearTransform() {
+    return AndTransformer<BitFieldsT>{~(static_cast<U>(kMask) << kBitOffset)};
+  }
+
+  // Create a transform for bitwise-and
+  static AndTransformer<BitFieldsT> AndTransform(V value) {
+    assert((value & ~kMask) == 0);
+    return AndTransformer<BitFieldsT>{
+        ~(static_cast<U>(value ^ kMask) << kBitOffset)};
+  }
+
+  // Create a transform for bitwise-or
+  static OrTransformer<BitFieldsT> OrTransform(V value) {
+    assert((value & ~kMask) == 0);
+    return OrTransformer<BitFieldsT>{static_cast<U>(value) << kBitOffset};
   }
 
   // Create a transform for adding a particular value, but with the precondition
   // that adding the value will not overflow the field. This applies for fields
   // that do not include the top bit of the underlying representation. Can be
   // combined with other additive transforms for other fields.
-  static AddTransform<BitFieldsT> PlusTransformPromiseNoOverflow(V value) {
+  static AddTransformer<BitFieldsT> PlusTransformPromiseNoOverflow(V value) {
     static_assert(!kIncludesTopBit);
-    AddTransform<BitFieldsT> rv{static_cast<U>(value) << kBitOffset};
+    AddTransformer<BitFieldsT> rv{static_cast<U>(value) << kBitOffset};
 #ifndef NDEBUG
     rv.preconditions.push_back(
         {static_cast<U>(kMask) << kBitOffset, rv.to_add});
@@ -281,9 +301,9 @@ struct UnsignedBitField {
   // in that field. This applies for fields that include the top bit of the
   // underlying representation. Can be combined with other additive transforms
   // for other fields.
-  static AddTransform<BitFieldsT> PlusTransformIgnoreOverflow(V value) {
+  static AddTransformer<BitFieldsT> PlusTransformIgnoreOverflow(V value) {
     static_assert(kIncludesTopBit);
-    AddTransform<BitFieldsT> rv{static_cast<U>(value) << kBitOffset};
+    AddTransformer<BitFieldsT> rv{static_cast<U>(value) << kBitOffset};
     return rv;
   }
 
@@ -292,9 +312,9 @@ struct UnsignedBitField {
   // applies for fields that do not include the top bit of the underlying
   // representation. Can be combined with other additive transforms for other
   // fields.
-  static AddTransform<BitFieldsT> MinusTransformPromiseNoUnderflow(V value) {
+  static AddTransformer<BitFieldsT> MinusTransformPromiseNoUnderflow(V value) {
     static_assert(!kIncludesTopBit);
-    AddTransform<BitFieldsT> rv{U{0} - (static_cast<U>(value) << kBitOffset)};
+    AddTransformer<BitFieldsT> rv{U{0} - (static_cast<U>(value) << kBitOffset)};
 #ifndef NDEBUG
     rv.preconditions.push_back(
         {static_cast<U>(kMask) << kBitOffset, rv.to_add});
@@ -306,17 +326,18 @@ struct UnsignedBitField {
   // underflow in that field. This applies for fields that include the top bit
   // of the underlying representation. Can be combined with other additive
   // transforms for other fields.
-  static AddTransform<BitFieldsT> MinusTransformIgnoreUnderflow(V value) {
+  static AddTransformer<BitFieldsT> MinusTransformIgnoreUnderflow(V value) {
     static_assert(kIncludesTopBit);
-    AddTransform<BitFieldsT> rv{U{0} - (static_cast<U>(value) << kBitOffset)};
+    AddTransformer<BitFieldsT> rv{U{0} - (static_cast<U>(value) << kBitOffset)};
     return rv;
   }
 };
 
-// A handy wrapper for a relaxed atomic on some BitFields type (unlike
-// RelaxedAtomic for arithmetic types). For encapsulation, usual arithmetic
-// atomic operations are only available by calling Apply[Relaxed]() on
-// Transforms returned from field classes. Extending an example from BitFields:
+// A handy wrapper for a relaxed atomic on some BitFields type, like
+// RelaxedAtomic but without direct arithmetic operations. For encapsulation,
+// usual arithmetic atomic operations are only available by calling
+// ApplyRelaxed() on Transforms returned from field classes. Extending an
+// example from BitFields:
 //
 // auto transform = Field2::ClearTransform() + Field4::ClearTransform();
 // MyState old_state;
@@ -347,22 +368,22 @@ class RelaxedBitFieldsAtomic {
     return BitFieldsT{
         v_.exchange(desired.underlying, std::memory_order_relaxed)};
   }
-  void ApplyRelaxed(OrTransform<BitFieldsT> transform,
+  void ApplyRelaxed(const OrTransformer<BitFieldsT>& transform,
                     BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
     ApplyImpl<std::memory_order_relaxed>(transform, before, after);
   }
-  void ApplyRelaxed(AndTransform<BitFieldsT> transform,
+  void ApplyRelaxed(const AndTransformer<BitFieldsT>& transform,
                     BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
     ApplyImpl<std::memory_order_relaxed>(transform, before, after);
   }
-  void ApplyRelaxed(AddTransform<BitFieldsT> transform,
+  void ApplyRelaxed(const AddTransformer<BitFieldsT>& transform,
                     BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
     ApplyImpl<std::memory_order_relaxed>(transform, before, after);
   }
 
  protected:  // fns
   template <std::memory_order kOrder>
-  void ApplyImpl(OrTransform<BitFieldsT> transform,
+  void ApplyImpl(const OrTransformer<BitFieldsT>& transform,
                  BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
     U before_val = v_.fetch_or(transform.to_or, kOrder);
     if (before) {
@@ -373,7 +394,7 @@ class RelaxedBitFieldsAtomic {
     }
   }
   template <std::memory_order kOrder>
-  void ApplyImpl(AndTransform<BitFieldsT> transform,
+  void ApplyImpl(const AndTransformer<BitFieldsT>& transform,
                  BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
     U before_val = v_.fetch_and(transform.to_and, kOrder);
     if (before) {
@@ -384,7 +405,7 @@ class RelaxedBitFieldsAtomic {
     }
   }
   template <std::memory_order kOrder>
-  void ApplyImpl(AddTransform<BitFieldsT> transform,
+  void ApplyImpl(const AddTransformer<BitFieldsT>& transform,
                  BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
     U before_val = v_.fetch_add(transform.to_add, kOrder);
     transform.AssertPreconditions(before_val);
@@ -401,14 +422,15 @@ class RelaxedBitFieldsAtomic {
 };
 
 // A handy wrapper for an aquire-release atomic (also relaxed semantics
-// available) on some BitFields type. See RelaxedBitFieldsAtomic for more info.
+// available) on some BitFields type. See RelaxedBitFieldsAtomic and
+// Atomic in atomic.h for more info.
 template <typename BitFieldsT>
-class AcqRelBitFieldsAtomic : public RelaxedBitFieldsAtomic<BitFieldsT> {
+class BitFieldsAtomic : public RelaxedBitFieldsAtomic<BitFieldsT> {
  public:
   using Base = RelaxedBitFieldsAtomic<BitFieldsT>;
   using U = typename BitFieldsT::U;
 
-  explicit AcqRelBitFieldsAtomic(BitFieldsT initial = {}) : Base(initial) {}
+  explicit BitFieldsAtomic(BitFieldsT initial = {}) : Base(initial) {}
 
   void Store(BitFieldsT desired) {
     Base::v_.store(desired.underlying, std::memory_order_release);
@@ -428,18 +450,18 @@ class AcqRelBitFieldsAtomic : public RelaxedBitFieldsAtomic<BitFieldsT> {
     return BitFieldsT{
         Base::v_.exchange(desired.underlying, std::memory_order_acq_rel)};
   }
-  void Apply(OrTransform<BitFieldsT> transform, BitFieldsT* before = nullptr,
-             BitFieldsT* after = nullptr) {
+  void Apply(const OrTransformer<BitFieldsT>& transform,
+             BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
     Base::template ApplyImpl<std::memory_order_acq_rel>(transform, before,
                                                         after);
   }
-  void Apply(AndTransform<BitFieldsT> transform, BitFieldsT* before = nullptr,
-             BitFieldsT* after = nullptr) {
+  void Apply(const AndTransformer<BitFieldsT>& transform,
+             BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
     Base::template ApplyImpl<std::memory_order_acq_rel>(transform, before,
                                                         after);
   }
-  void Apply(AddTransform<BitFieldsT> transform, BitFieldsT* before = nullptr,
-             BitFieldsT* after = nullptr) {
+  void Apply(const AddTransformer<BitFieldsT>& transform,
+             BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
     Base::template ApplyImpl<std::memory_order_acq_rel>(transform, before,
                                                         after);
   }

package/deps/rocksdb/rocksdb/util/bloom_impl.h

@@ -198,13 +198,13 @@ class FastLocalBloomImpl {
   }
 
   static inline void AddHash(uint32_t h1, uint32_t h2, uint32_t len_bytes,
-                             int num_probes, char *data) {
+                             int num_probes, char* data) {
     uint32_t bytes_to_cache_line = FastRange32(h1, len_bytes >> 6) << 6;
     AddHashPrepared(h2, num_probes, data + bytes_to_cache_line);
   }
 
   static inline void AddHashPrepared(uint32_t h2, int num_probes,
-                                     char *data_at_cache_line) {
+                                     char* data_at_cache_line) {
     uint32_t h = h2;
     for (int i = 0; i < num_probes; ++i, h *= uint32_t{0x9e3779b9}) {
       // 9-bit address within 512 bit cache line
@@ -214,8 +214,8 @@ class FastLocalBloomImpl {
   }
 
   static inline void PrepareHash(uint32_t h1, uint32_t len_bytes,
-                                 const char *data,
-                                 uint32_t /*out*/ *byte_offset) {
+                                 const char* data,
+                                 uint32_t /*out*/* byte_offset) {
     uint32_t bytes_to_cache_line = FastRange32(h1, len_bytes >> 6) << 6;
     PREFETCH(data + bytes_to_cache_line, 0 /* rw */, 1 /* locality */);
     PREFETCH(data + bytes_to_cache_line + 63, 0 /* rw */, 1 /* locality */);
@@ -223,13 +223,13 @@ class FastLocalBloomImpl {
   }
 
   static inline bool HashMayMatch(uint32_t h1, uint32_t h2, uint32_t len_bytes,
-                                  int num_probes, const char *data) {
+                                  int num_probes, const char* data) {
     uint32_t bytes_to_cache_line = FastRange32(h1, len_bytes >> 6) << 6;
     return HashMayMatchPrepared(h2, num_probes, data + bytes_to_cache_line);
   }
 
   static inline bool HashMayMatchPrepared(uint32_t h2, int num_probes,
-                                          const char *data_at_cache_line) {
+                                          const char* data_at_cache_line) {
     uint32_t h = h2;
 #ifdef __AVX2__
     int rem_probes = num_probes;
@@ -277,8 +277,8 @@ class FastLocalBloomImpl {
       //                            /*bytes / i32*/ 4);
       // END Option 1
       // Potentially unaligned as we're not *always* cache-aligned -> loadu
-      const __m256i *mm_data =
-          reinterpret_cast<const __m256i *>(data_at_cache_line);
+      const __m256i* mm_data =
+          reinterpret_cast<const __m256i*>(data_at_cache_line);
       __m256i lower = _mm256_loadu_si256(mm_data);
       __m256i upper = _mm256_loadu_si256(mm_data + 1);
       // Option 2: AVX512VL permute hack
@@ -362,7 +362,7 @@ class LegacyNoLocalityBloomImpl {
   }
 
   static inline void AddHash(uint32_t h, uint32_t total_bits, int num_probes,
-                             char *data) {
+                             char* data) {
     const uint32_t delta = (h >> 17) | (h << 15);  // Rotate right 17 bits
     for (int i = 0; i < num_probes; i++) {
       const uint32_t bitpos = h % total_bits;
@@ -372,7 +372,7 @@ class LegacyNoLocalityBloomImpl {
   }
 
   static inline bool HashMayMatch(uint32_t h, uint32_t total_bits,
-                                  int num_probes, const char *data) {
+                                  int num_probes, const char* data) {
     const uint32_t delta = (h >> 17) | (h << 15);  // Rotate right 17 bits
     for (int i = 0; i < num_probes; i++) {
       const uint32_t bitpos = h % total_bits;
@@ -430,10 +430,10 @@ class LegacyLocalityBloomImpl {
   }
 
   static inline void AddHash(uint32_t h, uint32_t num_lines, int num_probes,
-                             char *data, int log2_cache_line_bytes) {
+                             char* data, int log2_cache_line_bytes) {
     const int log2_cache_line_bits = log2_cache_line_bytes + 3;
 
-    char *data_at_offset =
+    char* data_at_offset =
         data + (GetLine(h, num_lines) << log2_cache_line_bytes);
     const uint32_t delta = (h >> 17) | (h << 15);
     for (int i = 0; i < num_probes; ++i) {
@@ -448,8 +448,8 @@ class LegacyLocalityBloomImpl {
   }
 
   static inline void PrepareHashMayMatch(uint32_t h, uint32_t num_lines,
-                                         const char *data,
-                                         uint32_t /*out*/ *byte_offset,
+                                         const char* data,
+                                         uint32_t /*out*/* byte_offset,
                                          int log2_cache_line_bytes) {
     uint32_t b = GetLine(h, num_lines) << log2_cache_line_bytes;
     PREFETCH(data + b, 0 /* rw */, 1 /* locality */);
@@ -459,14 +459,14 @@ class LegacyLocalityBloomImpl {
   }
 
   static inline bool HashMayMatch(uint32_t h, uint32_t num_lines,
-                                  int num_probes, const char *data,
+                                  int num_probes, const char* data,
                                   int log2_cache_line_bytes) {
     uint32_t b = GetLine(h, num_lines) << log2_cache_line_bytes;
     return HashMayMatchPrepared(h, num_probes, data + b, log2_cache_line_bytes);
   }
 
   static inline bool HashMayMatchPrepared(uint32_t h, int num_probes,
-                                          const char *data_at_offset,
+                                          const char* data_at_offset,
                                           int log2_cache_line_bytes) {
     const int log2_cache_line_bits = log2_cache_line_bytes + 3;
 

package/deps/rocksdb/rocksdb/util/coding.h

@@ -18,6 +18,7 @@
 #pragma once
 #include <algorithm>
 #include <string>
+#include <type_traits>
 
 #include "port/port.h"
 #include "rocksdb/slice.h"
@@ -38,10 +39,9 @@ const uint32_t kMaxVarint64Length = 10;
 void PutFixed16(std::string* dst, uint16_t value);
 void PutFixed32(std::string* dst, uint32_t value);
 void PutFixed64(std::string* dst, uint64_t value);
-void PutVarint32(std::string* dst, uint32_t value);
-void PutVarint32Varint32(std::string* dst, uint32_t value1, uint32_t value2);
-void PutVarint32Varint32Varint32(std::string* dst, uint32_t value1,
-                                 uint32_t value2, uint32_t value3);
+
+template <typename... Args>
+void PutVarint32(std::string* dst, Args... args);
 void PutVarint64(std::string* dst, uint64_t value);
 void PutVarint64Varint64(std::string* dst, uint64_t value1, uint64_t value2);
 void PutVarint32Varint64(std::string* dst, uint32_t value1, uint64_t value2);
@@ -149,25 +149,14 @@ inline void PutFixed64(std::string* dst, uint64_t value) {
   }
 }
 
-inline void PutVarint32(std::string* dst, uint32_t v) {
-  char buf[5];
-  char* ptr = EncodeVarint32(buf, v);
-  dst->append(buf, static_cast<size_t>(ptr - buf));
-}
-
-inline void PutVarint32Varint32(std::string* dst, uint32_t v1, uint32_t v2) {
-  char buf[10];
-  char* ptr = EncodeVarint32(buf, v1);
-  ptr = EncodeVarint32(ptr, v2);
-  dst->append(buf, static_cast<size_t>(ptr - buf));
-}
-
-inline void PutVarint32Varint32Varint32(std::string* dst, uint32_t v1,
-                                        uint32_t v2, uint32_t v3) {
-  char buf[15];
-  char* ptr = EncodeVarint32(buf, v1);
-  ptr = EncodeVarint32(ptr, v2);
-  ptr = EncodeVarint32(ptr, v3);
+template <typename... Args>
+inline void PutVarint32(std::string* dst, Args... args) {
+  static_assert((std::is_convertible_v<Args, uint32_t> && ...),
+                "All arguments must be convertible to uint32_t");
+  constexpr size_t kMaxBytesPerVarint32 = 5;
+  char buf[sizeof...(args) * kMaxBytesPerVarint32];
+  char* ptr = buf;
+  ((ptr = EncodeVarint32(ptr, static_cast<uint32_t>(args))), ...);
   dst->append(buf, static_cast<size_t>(ptr - buf));
 }
 
@@ -355,8 +344,7 @@ __attribute__((__no_sanitize__("alignment")))
 __attribute__((__no_sanitize_undefined__))
 #endif
 #endif
-inline void
-PutUnaligned(T* memory, const T& value) {
+inline void PutUnaligned(T* memory, const T& value) {
 #if defined(PLATFORM_UNALIGNED_ACCESS_NOT_ALLOWED)
   char* nonAlignedMemory = reinterpret_cast<char*>(memory);
   memcpy(nonAlignedMemory, reinterpret_cast<const char*>(&value), sizeof(T));
@@ -373,8 +361,7 @@ __attribute__((__no_sanitize__("alignment")))
 __attribute__((__no_sanitize_undefined__))
 #endif
 #endif
-inline void
-GetUnaligned(const T* memory, T* value) {
+inline void GetUnaligned(const T* memory, T* value) {
 #if defined(PLATFORM_UNALIGNED_ACCESS_NOT_ALLOWED)
   char* nonAlignedMemory = reinterpret_cast<char*>(value);
   memcpy(nonAlignedMemory, reinterpret_cast<const char*>(memory), sizeof(T));