@nxtedition/rocksdb 8.2.0 → 8.2.2

package/deps/rocksdb/rocksdb/tools/db_bench_tool.cc

@@ -33,6 +33,7 @@
 #include <iostream>
 #include <memory>
 #include <mutex>
+#include <optional>
 #include <queue>
 #include <thread>
 #include <unordered_map>
@@ -41,7 +42,7 @@
 #include "db/malloc_stats.h"
 #include "db/version_set.h"
 #include "monitoring/histogram.h"
-#include "monitoring/statistics.h"
+#include "monitoring/statistics_impl.h"
 #include "options/cf_options.h"
 #include "port/port.h"
 #include "port/stack_trace.h"
@@ -287,7 +288,7 @@ DEFINE_int32(bloom_locality, 0, "Control bloom filter probes locality");
 DEFINE_int64(seed, 0,
              "Seed base for random number generators. "
              "When 0 it is derived from the current time.");
-static int64_t seed_base;
+static std::optional<int64_t> seed_base;
 
 DEFINE_int32(threads, 1, "Number of concurrent threads to run.");
 
@@ -548,7 +549,7 @@ DEFINE_bool(universal_allow_trivial_move, false,
 DEFINE_bool(universal_incremental, false,
             "Enable incremental compactions in universal compaction.");
 
-DEFINE_int64(cache_size, 8 << 20,  // 8MB
+DEFINE_int64(cache_size, 32 << 20,  // 32MB
              "Number of bytes to use as a cache of uncompressed data");
 
 DEFINE_int32(cache_numshardbits, -1,
@@ -569,7 +570,7 @@ DEFINE_string(cache_type, "lru_cache", "Type of block cache.");
 DEFINE_bool(use_compressed_secondary_cache, false,
             "Use the CompressedSecondaryCache as the secondary cache.");
 
-DEFINE_int64(compressed_secondary_cache_size, 8 << 20,  // 8MB
+DEFINE_int64(compressed_secondary_cache_size, 32 << 20,  // 32MB
              "Number of bytes to use as a cache of data");
 
 DEFINE_int32(compressed_secondary_cache_numshardbits, 6,
@@ -1725,6 +1726,10 @@ DEFINE_uint32(
     "This options determines the size of such checksums. "
     "Supported values: 0, 1, 2, 4, 8.");
 
+DEFINE_uint32(block_protection_bytes_per_key, 0,
+              "Enable block per key-value checksum protection. "
+              "Supported values: 0, 1, 2, 4, 8.");
+
 DEFINE_bool(build_info, false,
             "Print the build info via GetRocksBuildInfoAsString");
 
@@ -2602,7 +2607,7 @@ struct ThreadState {
   SharedState* shared;
 
   explicit ThreadState(int index, int my_seed)
-      : tid(index), rand(seed_base + my_seed) {}
+      : tid(index), rand(*seed_base + my_seed) {}
 };
 
 class Duration {
@@ -2998,6 +3003,11 @@ class Benchmark {
     return allocator;
   }
 
+  static int32_t GetCacheHashSeed() {
+    // For a fixed Cache seed, need a non-negative int32
+    return static_cast<int32_t>(*seed_base) & 0x7fffffff;
+  }
+
   static std::shared_ptr<Cache> NewCache(int64_t capacity) {
     if (capacity <= 0) {
       return nullptr;
@@ -3006,17 +3016,19 @@ class Benchmark {
       fprintf(stderr, "Old clock cache implementation has been removed.\n");
       exit(1);
     } else if (FLAGS_cache_type == "hyper_clock_cache") {
-      return HyperClockCacheOptions(static_cast<size_t>(capacity),
-                                    FLAGS_block_size /*estimated_entry_charge*/,
-                                    FLAGS_cache_numshardbits)
-          .MakeSharedCache();
+      HyperClockCacheOptions hcco{
+          static_cast<size_t>(capacity),
+          static_cast<size_t>(FLAGS_block_size) /*estimated_entry_charge*/,
+          FLAGS_cache_numshardbits};
+      hcco.hash_seed = GetCacheHashSeed();
+      return hcco.MakeSharedCache();
     } else if (FLAGS_cache_type == "lru_cache") {
       LRUCacheOptions opts(
           static_cast<size_t>(capacity), FLAGS_cache_numshardbits,
           false /*strict_capacity_limit*/, FLAGS_cache_high_pri_pool_ratio,
           GetCacheAllocator(), kDefaultToAdaptiveMutex,
           kDefaultCacheMetadataChargePolicy, FLAGS_cache_low_pri_pool_ratio);
-
+      opts.hash_seed = GetCacheHashSeed();
       if (!FLAGS_secondary_cache_uri.empty()) {
         Status s = SecondaryCache::CreateFromString(
             ConfigOptions(), FLAGS_secondary_cache_uri, &secondary_cache);
@@ -3047,7 +3059,7 @@ class Benchmark {
             NewCompressedSecondaryCache(secondary_cache_opts);
       }
 
-      return NewLRUCache(opts);
+      return opts.MakeSharedCache();
     } else {
       fprintf(stderr, "Cache type not supported.");
       exit(1);
@@ -4565,6 +4577,8 @@ class Benchmark {
     }
     options.memtable_protection_bytes_per_key =
         FLAGS_memtable_protection_bytes_per_key;
+    options.block_protection_bytes_per_key =
+        FLAGS_block_protection_bytes_per_key;
   }
 
   void InitializeOptionsGeneral(Options* opts) {
@@ -4590,7 +4604,7 @@ class Benchmark {
       if (FLAGS_cache_size > 0) {
         // This violates this function's rules on when to set options. But we
         // have to do it because the case of unconfigured block cache in OPTIONS
-        // file is indistinguishable (it is sanitized to 8MB by this point, not
+        // file is indistinguishable (it is sanitized to 32MB by this point, not
         // nullptr), and our regression tests assume this will be the shared
         // block cache, even with OPTIONS file provided.
         table_options->block_cache = cache_;
@@ -4885,7 +4899,7 @@ class Benchmark {
           values_[i] = i;
         }
         RandomShuffle(values_.begin(), values_.end(),
-                      static_cast<uint32_t>(seed_base));
+                      static_cast<uint32_t>(*seed_base));
       }
     }
 
@@ -4997,7 +5011,7 @@ class Benchmark {
     // Default_random_engine provides slightly
     // improved throughput over mt19937.
     std::default_random_engine overwrite_gen{
-        static_cast<unsigned int>(seed_base)};
+        static_cast<unsigned int>(*seed_base)};
     std::bernoulli_distribution overwrite_decider(p);
 
     // Inserted key window is filled with the last N
@@ -5007,7 +5021,7 @@ class Benchmark {
     // - random access is O(1)
     // - insertion/removal at beginning/end is also O(1).
     std::deque<int64_t> inserted_key_window;
-    Random64 reservoir_id_gen(seed_base);
+    Random64 reservoir_id_gen(*seed_base);
 
     // --- Variables used in disposable/persistent keys simulation:
     // The following variables are used when
@@ -5044,7 +5058,7 @@ class Benchmark {
         ErrorExit();
       }
     }
-    Random rnd_disposable_entry(static_cast<uint32_t>(seed_base));
+    Random rnd_disposable_entry(static_cast<uint32_t>(*seed_base));
     std::string random_value;
     // Queue that stores scheduled timestamp of disposable entries deletes,
     // along with starting index of disposable entry keys to delete.
@@ -6978,6 +6992,8 @@ class Benchmark {
 
       thread->stats.FinishedOps(&db_, db_.db, 1, kSeek);
     }
+    (void)num_seek_to_first;
+    (void)num_next;
     delete iter;
   }
 
@@ -8508,7 +8524,7 @@ int db_bench_tool(int argc, char** argv) {
     uint64_t now = FLAGS_env->GetSystemClock()->NowMicros();
     seed_base = static_cast<int64_t>(now);
     fprintf(stdout, "Set seed to %" PRIu64 " because --seed was 0\n",
-            seed_base);
+            *seed_base);
   } else {
     seed_base = FLAGS_seed;
   }

package/deps/rocksdb/rocksdb/tools/ldb_cmd.cc

@@ -4168,6 +4168,8 @@ UnsafeRemoveSstFileCommand::UnsafeRemoveSstFileCommand(
 }
 
 void UnsafeRemoveSstFileCommand::DoCommand() {
+  // TODO: plumb Env::IOActivity
+  const ReadOptions read_options;
   PrepareOptions();
 
   OfflineManifestWriter w(options_, db_path_);
@@ -4192,7 +4194,7 @@ void UnsafeRemoveSstFileCommand::DoCommand() {
     s = options_.env->GetFileSystem()->NewDirectory(db_path_, IOOptions(),
                                                     &db_dir, nullptr);
     if (s.ok()) {
-      s = w.LogAndApply(cfd, &edit, db_dir.get());
+      s = w.LogAndApply(read_options, cfd, &edit, db_dir.get());
     }
   }
 

package/deps/rocksdb/rocksdb/util/bloom_impl.h

@@ -17,7 +17,7 @@
 #include "rocksdb/slice.h"
 #include "util/hash.h"
 
-#ifdef HAVE_AVX2
+#ifdef __AVX2__
 #include <immintrin.h>
 #endif
 
@@ -231,7 +231,7 @@ class FastLocalBloomImpl {
   static inline bool HashMayMatchPrepared(uint32_t h2, int num_probes,
                                           const char *data_at_cache_line) {
     uint32_t h = h2;
-#ifdef HAVE_AVX2
+#ifdef __AVX2__
     int rem_probes = num_probes;
 
     // NOTE: For better performance for num_probes in {1, 2, 9, 10, 17, 18,

package/deps/rocksdb/rocksdb/util/compression.h

@@ -20,7 +20,7 @@
 #endif  // ROCKSDB_MALLOC_USABLE_SIZE
 #include <string>
 
-#include "memory/memory_allocator.h"
+#include "memory/memory_allocator_impl.h"
 #include "rocksdb/options.h"
 #include "rocksdb/table.h"
 #include "table/block_based/block_type.h"

package/deps/rocksdb/rocksdb/util/crc32c.cc

@@ -15,10 +15,6 @@
 
 #include <array>
 #include <utility>
-#ifdef HAVE_SSE42
-#include <nmmintrin.h>
-#include <wmmintrin.h>
-#endif
 
 #include "port/lang.h"
 #include "util/coding.h"
@@ -50,6 +46,13 @@
 
 #endif
 
+ASSERT_FEATURE_COMPAT_HEADER();
+
+#ifdef __SSE4_2__
+#include <nmmintrin.h>
+#include <wmmintrin.h>
+#endif
+
 #if defined(HAVE_ARM64_CRC)
 bool pmull_runtime_flag = false;
 #endif
@@ -107,6 +110,7 @@ static const uint32_t table0_[256] = {
     0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, 0x34f4f86a, 0xc69f7b69,
     0xd5cf889d, 0x27a40b9e, 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e,
     0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351};
+#ifndef __SSE4_2__
 static const uint32_t table1_[256] = {
     0x00000000, 0x13a29877, 0x274530ee, 0x34e7a899, 0x4e8a61dc, 0x5d28f9ab,
     0x69cf5132, 0x7a6dc945, 0x9d14c3b8, 0x8eb65bcf, 0xba51f356, 0xa9f36b21,
@@ -244,14 +248,10 @@ static const uint32_t table3_[256] = {
 static inline uint32_t LE_LOAD32(const uint8_t* p) {
   return DecodeFixed32(reinterpret_cast<const char*>(p));
 }
+#endif  // !__SSE4_2__
 
-#if defined(HAVE_SSE42) && (defined(__LP64__) || defined(_WIN64))
-static inline uint64_t LE_LOAD64(const uint8_t* p) {
-  return DecodeFixed64(reinterpret_cast<const char*>(p));
-}
-#endif
-
-static inline void Slow_CRC32(uint64_t* l, uint8_t const** p) {
+static inline void DefaultCRC32(uint64_t* l, uint8_t const** p) {
+#ifndef __SSE4_2__
   uint32_t c = static_cast<uint32_t>(*l ^ LE_LOAD32(*p));
   *p += 4;
   *l = table3_[c & 0xff] ^ table2_[(c >> 8) & 0xff] ^
@@ -261,16 +261,8 @@ static inline void Slow_CRC32(uint64_t* l, uint8_t const** p) {
   *p += 4;
   *l = table3_[c & 0xff] ^ table2_[(c >> 8) & 0xff] ^
        table1_[(c >> 16) & 0xff] ^ table0_[c >> 24];
-}
-
-#if (!(defined(HAVE_POWER8) && defined(HAS_ALTIVEC))) && \
-        (!defined(HAVE_ARM64_CRC)) ||                    \
-    defined(NO_THREEWAY_CRC32C)
-static inline void Fast_CRC32(uint64_t* l, uint8_t const** p) {
-#ifndef HAVE_SSE42
-  Slow_CRC32(l, p);
 #elif defined(__LP64__) || defined(_WIN64)
-  *l = _mm_crc32_u64(*l, LE_LOAD64(*p));
+  *l = _mm_crc32_u64(*l, DecodeFixed64(reinterpret_cast<const char*>(*p)));
   *p += 8;
 #else
   *l = _mm_crc32_u32(static_cast<unsigned int>(*l), LE_LOAD32(*p));
@@ -279,7 +271,6 @@ static inline void Fast_CRC32(uint64_t* l, uint8_t const** p) {
   *p += 4;
 #endif
 }
-#endif
 
 template <void (*CRC32)(uint64_t*, uint8_t const**)>
 uint32_t ExtendImpl(uint32_t crc, const char* buf, size_t size) {
@@ -324,48 +315,6 @@ uint32_t ExtendImpl(uint32_t crc, const char* buf, size_t size) {
   return static_cast<uint32_t>(l ^ 0xffffffffu);
 }
 
-// Detect if ARM64 CRC or not.
-#ifndef HAVE_ARM64_CRC
-// Detect if SS42 or not.
-#ifndef HAVE_POWER8
-
-static bool isSSE42() {
-#ifndef HAVE_SSE42
-  return false;
-#elif defined(__GNUC__) && defined(__x86_64__) && !defined(IOS_CROSS_COMPILE)
-  uint32_t c_;
-  __asm__("cpuid" : "=c"(c_) : "a"(1) : "ebx", "edx");
-  return c_ & (1U << 20);  // copied from CpuId.h in Folly. Test SSE42
-#elif defined(_WIN64)
-  int info[4];
-  __cpuidex(info, 0x00000001, 0);
-  return (info[2] & ((int)1 << 20)) != 0;
-#else
-  return false;
-#endif
-}
-
-static bool isPCLMULQDQ() {
-#ifndef HAVE_SSE42
-  // in build_detect_platform we set this macro when both SSE42 and PCLMULQDQ
-  // are supported by compiler
-  return false;
-#elif defined(__GNUC__) && defined(__x86_64__) && !defined(IOS_CROSS_COMPILE)
-  uint32_t c_;
-  __asm__("cpuid" : "=c"(c_) : "a"(1) : "ebx", "edx");
-  return c_ & (1U << 1);  // PCLMULQDQ is in bit 1 (not bit 0)
-#elif defined(_WIN64)
-  int info[4];
-  __cpuidex(info, 0x00000001, 0);
-  return (info[2] & ((int)1 << 1)) != 0;
-#else
-  return false;
-#endif
-}
-
-#endif  // HAVE_POWER8
-#endif  // HAVE_ARM64_CRC
-
 using Function = uint32_t (*)(uint32_t, const char*, size_t);
 
 #if defined(HAVE_POWER8) && defined(HAS_ALTIVEC)
@@ -436,7 +385,9 @@ std::string IsFastCrc32Supported() {
     arch = "Arm64";
   }
 #else
-  has_fast_crc = isSSE42();
+#ifdef __SSE4_2__
+  has_fast_crc = true;
+#endif  // __SSE4_2__
   arch = "x86";
 #endif
   if (has_fast_crc) {
@@ -477,7 +428,7 @@ std::string IsFastCrc32Supported() {
  * <davejwatson@fb.com>
  *
  */
-#if defined HAVE_SSE42 && defined HAVE_PCLMUL
+#if defined(__SSE4_2__) && defined(__PCLMUL__)
 
 #define CRCtriplet(crc, buf, offset)                  \
   crc##0 = _mm_crc32_u64(crc##0, *(buf##0 + offset)); \
@@ -1152,34 +1103,24 @@ uint32_t crc32c_3way(uint32_t crc, const char* buf, size_t len) {
   }
 }
 
-#endif //HAVE_SSE42 && HAVE_PCLMUL
+#endif //__SSE4_2__ && __PCLMUL__
 
 static inline Function Choose_Extend() {
 #ifdef HAVE_POWER8
-  return isAltiVec() ? ExtendPPCImpl : ExtendImpl<Slow_CRC32>;
+  return isAltiVec() ? ExtendPPCImpl : ExtendImpl<DefaultCRC32>;
 #elif defined(HAVE_ARM64_CRC)
   if(crc32c_runtime_check()) {
     pmull_runtime_flag = crc32c_pmull_runtime_check();
     return ExtendARMImpl;
   } else {
-    return ExtendImpl<Slow_CRC32>;
+    return ExtendImpl<DefaultCRC32>;
   }
+#elif defined(__SSE4_2__) && defined(__PCLMUL__) && !defined NO_THREEWAY_CRC32C
+  // NOTE: runtime detection no longer supported on x86
+  (void)ExtendImpl<DefaultCRC32>;  // suppress unused warning
+  return crc32c_3way;
 #else
-  if (isSSE42()) {
-    if (isPCLMULQDQ()) {
-#if (defined HAVE_SSE42 && defined HAVE_PCLMUL) && !defined NO_THREEWAY_CRC32C
-      return crc32c_3way;
-#else
-    return ExtendImpl<Fast_CRC32>; // Fast_CRC32 will check HAVE_SSE42 itself
-#endif
-    }
-    else {  // no runtime PCLMULQDQ support but has SSE42 support
-      return ExtendImpl<Fast_CRC32>;
-    }
-  } // end of isSSE42()
-  else {
-    return ExtendImpl<Slow_CRC32>;
-  }
+  return ExtendImpl<DefaultCRC32>;
 #endif
 }
 

package/deps/rocksdb/rocksdb/util/crc32c_arm64.cc

@@ -23,10 +23,10 @@
 #include <sys/sysctl.h>
 #endif
 #if defined(__OpenBSD__)
-#include <sys/types.h>
-#include <sys/sysctl.h>
-#include <machine/cpu.h>
 #include <machine/armreg.h>
+#include <machine/cpu.h>
+#include <sys/sysctl.h>
+#include <sys/types.h>
 #endif
 
 #ifdef HAVE_ARM64_CRYPTO
@@ -67,13 +67,12 @@ uint32_t crc32c_runtime_check(void) {
   return r == 1;
 #elif defined(__OpenBSD__)
   int r = 0;
-  const int isar0_mib[] = { CTL_MACHDEP, CPU_ID_AA64ISAR0 };
+  const int isar0_mib[] = {CTL_MACHDEP, CPU_ID_AA64ISAR0};
   uint64_t isar0;
   size_t len = sizeof(isar0);
 
   if (sysctl(isar0_mib, 2, &isar0, &len, NULL, 0) != -1) {
-      if (ID_AA64ISAR0_CRC32(isar0) >= ID_AA64ISAR0_CRC32_BASE)
-        r = 1;
+    if (ID_AA64ISAR0_CRC32(isar0) >= ID_AA64ISAR0_CRC32_BASE) r = 1;
   }
   return r;
 #else
@@ -94,13 +93,12 @@ bool crc32c_pmull_runtime_check(void) {
   return true;
 #elif defined(__OpenBSD__)
   bool r = false;
-  const int isar0_mib[] = { CTL_MACHDEP, CPU_ID_AA64ISAR0 };
+  const int isar0_mib[] = {CTL_MACHDEP, CPU_ID_AA64ISAR0};
   uint64_t isar0;
   size_t len = sizeof(isar0);
 
   if (sysctl(isar0_mib, 2, &isar0, &len, NULL, 0) != -1) {
-      if (ID_AA64ISAR0_AES(isar0) >= ID_AA64ISAR0_AES_PMULL)
-        r = true;
+    if (ID_AA64ISAR0_AES(isar0) >= ID_AA64ISAR0_AES_PMULL) r = true;
   }
   return r;
 #else

package/deps/rocksdb/rocksdb/util/file_checksum_helper.cc

@@ -98,6 +98,8 @@ Status GetFileChecksumsFromManifest(Env* src_env, const std::string& abs_path,
     return Status::InvalidArgument("checksum_list is nullptr");
   }
   assert(checksum_list);
+  // TODO: plumb Env::IOActivity
+  const ReadOptions read_options;
   checksum_list->reset();
   Status s;
 
@@ -125,7 +127,8 @@ Status GetFileChecksumsFromManifest(Env* src_env, const std::string& abs_path,
   reporter.status_ptr = &s;
   log::Reader reader(nullptr, std::move(file_reader), &reporter,
                      true /* checksum */, 0 /* log_number */);
-  FileChecksumRetriever retriever(manifest_file_size, *checksum_list);
+  FileChecksumRetriever retriever(read_options, manifest_file_size,
+                                  *checksum_list);
   retriever.Iterate(reader, &s);
   assert(!retriever.status().ok() ||
          manifest_file_size == std::numeric_limits<uint64_t>::max() ||

package/deps/rocksdb/rocksdb/util/filter_bench.cc

@@ -728,7 +728,7 @@ double FilterBench::RandomQueryTest(uint32_t inside_threshold, bool dry_run,
                 batch_slices[i],
                 /*no_io=*/false, /*const_ikey_ptr=*/nullptr,
                 /*get_context=*/nullptr,
-                /*lookup_context=*/nullptr, Env::IO_TOTAL);
+                /*lookup_context=*/nullptr, ROCKSDB_NAMESPACE::ReadOptions());
           }
         } else {
           if (dry_run) {

package/deps/rocksdb/rocksdb/util/gflags_compat.h

@@ -15,16 +15,15 @@
 #endif
 
 #ifndef DEFINE_uint32
-// DEFINE_uint32 does not appear in older versions of gflags. This should be
-// a sane definition for those versions.
+// DEFINE_uint32 / DECLARE_uint32 do not appear in older versions of gflags.
+// These should be sane definitions for those versions.
 #include <cstdint>
-#define DEFINE_uint32(name, val, txt)             \
-  namespace gflags_compat {                       \
-  DEFINE_int32(name, val, txt);                   \
-  }                                               \
-  std::reference_wrapper<uint32_t> FLAGS_##name = \
-      std::ref(*reinterpret_cast<uint32_t *>(&gflags_compat::FLAGS_##name));
+#define DEFINE_uint32(name, val, txt) \
+  namespace gflags_compat {           \
+  DEFINE_int32(name, val, txt);       \
+  }                                   \
+  uint32_t &FLAGS_##name =            \
+      *reinterpret_cast<uint32_t *>(&gflags_compat::FLAGS_##name);
 
-#define DECLARE_uint32(name) \
-  extern std::reference_wrapper<uint32_t> FLAGS_##name;
+#define DECLARE_uint32(name) extern uint32_t &FLAGS_##name;
 #endif  // !DEFINE_uint32

package/deps/rocksdb/rocksdb/util/math.h

@@ -13,8 +13,11 @@
 #include <cstdint>
 #include <type_traits>
 
+#include "port/lang.h"
 #include "rocksdb/rocksdb_namespace.h"
 
+ASSERT_FEATURE_COMPAT_HEADER();
+
 namespace ROCKSDB_NAMESPACE {
 
 // Fast implementation of floor(log2(v)). Undefined for 0 or negative
@@ -145,27 +148,29 @@ inline int BitsSetToOne(T v) {
     constexpr auto mm = 8 * sizeof(uint32_t) - 1;
     // The bit mask is to neutralize sign extension on small signed types
     constexpr uint32_t m = (uint32_t{1} << ((8 * sizeof(T)) & mm)) - 1;
-#if defined(HAVE_SSE42) && (defined(_M_X64) || defined(_M_IX86))
+#if __POPCNT__
     return static_cast<int>(__popcnt(static_cast<uint32_t>(v) & m));
 #else
     return static_cast<int>(detail::BitsSetToOneFallback(v) & m);
-#endif
+#endif  // __POPCNT__
   } else if (sizeof(T) == sizeof(uint32_t)) {
-#if defined(HAVE_SSE42) && (defined(_M_X64) || defined(_M_IX86))
+#if __POPCNT__
     return static_cast<int>(__popcnt(static_cast<uint32_t>(v)));
 #else
     return detail::BitsSetToOneFallback(static_cast<uint32_t>(v));
-#endif
+#endif  // __POPCNT__
   } else {
-#if defined(HAVE_SSE42) && defined(_M_X64)
+#if __POPCNT__
+#ifdef _M_X64
     return static_cast<int>(__popcnt64(static_cast<uint64_t>(v)));
-#elif defined(HAVE_SSE42) && defined(_M_IX86)
+#else
     return static_cast<int>(
         __popcnt(static_cast<uint32_t>(static_cast<uint64_t>(v) >> 32) +
                  __popcnt(static_cast<uint32_t>(v))));
+#endif  // _M_X64
 #else
     return detail::BitsSetToOneFallback(static_cast<uint64_t>(v));
-#endif
+#endif  // __POPCNT__
   }
 #else
   static_assert(sizeof(T) <= sizeof(unsigned long long), "type too big");

package/deps/rocksdb/rocksdb/util/rate_limiter.cc

@@ -7,15 +7,14 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-#include "util/rate_limiter.h"
-
 #include <algorithm>
 
-#include "monitoring/statistics.h"
+#include "monitoring/statistics_impl.h"
 #include "port/port.h"
 #include "rocksdb/system_clock.h"
 #include "test_util/sync_point.h"
 #include "util/aligned_buffer.h"
+#include "util/rate_limiter_impl.h"
 
 namespace ROCKSDB_NAMESPACE {
 size_t RateLimiter::RequestToken(size_t bytes, size_t alignment,
@@ -38,11 +37,10 @@ size_t RateLimiter::RequestToken(size_t bytes, size_t alignment,
 // Pending request
 struct GenericRateLimiter::Req {
   explicit Req(int64_t _bytes, port::Mutex* _mu)
-      : request_bytes(_bytes), bytes(_bytes), cv(_mu), granted(false) {}
+      : request_bytes(_bytes), bytes(_bytes), cv(_mu) {}
   int64_t request_bytes;
   int64_t bytes;
   port::CondVar cv;
-  bool granted;
 };
 
 GenericRateLimiter::GenericRateLimiter(
@@ -137,12 +135,14 @@ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri,
 
   ++total_requests_[pri];
 
-  if (available_bytes_ >= bytes) {
-    // Refill thread assigns quota and notifies requests waiting on
-    // the queue under mutex. So if we get here, that means nobody
-    // is waiting?
-    available_bytes_ -= bytes;
-    total_bytes_through_[pri] += bytes;
+  if (available_bytes_ > 0) {
+    int64_t bytes_through = std::min(available_bytes_, bytes);
+    total_bytes_through_[pri] += bytes_through;
+    available_bytes_ -= bytes_through;
+    bytes -= bytes_through;
+  }
+
+  if (bytes == 0) {
     return;
   }
 
@@ -179,7 +179,7 @@ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri,
       // Whichever thread reaches here first performs duty (2) as described
       // above.
       RefillBytesAndGrantRequestsLocked();
-      if (r.granted) {
+      if (r.request_bytes == 0) {
         // If there is any remaining requests, make sure there exists at least
         // one candidate is awake for future duties by signaling a front request
         // of a queue.
@@ -202,13 +202,13 @@ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri,
         ++num_found;
       }
     }
-    if (r.granted) {
+    if (r.request_bytes == 0) {
       assert(num_found == 0);
     } else {
       assert(num_found == 1);
     }
 #endif  // NDEBUG
-  } while (!stop_ && !r.granted);
+  } while (!stop_ && r.request_bytes > 0);
 
   if (stop_) {
     // It is now in the clean-up of ~GenericRateLimiter().
@@ -265,9 +265,8 @@ void GenericRateLimiter::RefillBytesAndGrantRequestsLocked() {
   // Carry over the left over quota from the last period
   auto refill_bytes_per_period =
       refill_bytes_per_period_.load(std::memory_order_relaxed);
-  if (available_bytes_ < refill_bytes_per_period) {
-    available_bytes_ += refill_bytes_per_period;
-  }
+  assert(available_bytes_ == 0);
+  available_bytes_ = refill_bytes_per_period;
 
   std::vector<Env::IOPriority> pri_iteration_order =
       GeneratePriorityIterationOrderLocked();
@@ -292,7 +291,6 @@ void GenericRateLimiter::RefillBytesAndGrantRequestsLocked() {
       total_bytes_through_[current_pri] += next_req->bytes;
       queue->pop_front();
 
-      next_req->granted = true;
       // Quota granted, signal the thread to exit
       next_req->cv.Signal();
     }