@nxtedition/rocksdb 8.2.8 → 9.0.1

package/deps/rocksdb/rocksdb/util/math.h

@@ -9,6 +9,9 @@
 #ifdef _MSC_VER
 #include <intrin.h>
 #endif
+#ifdef __BMI2__
+#include <immintrin.h>
+#endif
 
 #include <cstdint>
 #include <type_traits>
@@ -20,11 +23,33 @@ ASSERT_FEATURE_COMPAT_HEADER();
 
 namespace ROCKSDB_NAMESPACE {
 
+// Fast implementation of extracting the bottom n bits of an integer.
+// To ensure fast implementation, undefined if n bits is full width or more.
+template <typename T>
+inline T BottomNBits(T v, int nbits) {
+  static_assert(std::is_integral_v<T>, "non-integral type");
+  static_assert(!std::is_reference_v<T>, "use std::remove_reference_t");
+  assert(nbits >= 0);
+  assert(nbits < int{8 * sizeof(T)});
+#ifdef __BMI2__
+  if constexpr (sizeof(T) <= 4) {
+    return static_cast<T>(_bzhi_u32(static_cast<uint32_t>(v), nbits));
+  }
+  if constexpr (sizeof(T) <= 8) {
+    return static_cast<T>(_bzhi_u64(static_cast<uint64_t>(v), nbits));
+  }
+#endif
+  // Newer compilers compile this down to bzhi on x86, but some older
+  // ones don't, thus the need for the intrinsic above.
+  return static_cast<T>(v & ((T{1} << nbits) - 1));
+}
+
 // Fast implementation of floor(log2(v)). Undefined for 0 or negative
 // numbers (in case of signed type).
 template <typename T>
 inline int FloorLog2(T v) {
-  static_assert(std::is_integral<T>::value, "non-integral type");
+  static_assert(std::is_integral_v<T>, "non-integral type");
+  static_assert(!std::is_reference_v<T>, "use std::remove_reference_t");
   assert(v > 0);
 #ifdef _MSC_VER
   static_assert(sizeof(T) <= sizeof(uint64_t), "type too big");
@@ -63,6 +88,8 @@ inline int FloorLog2(T v) {
 // Constexpr version of FloorLog2
 template <typename T>
 constexpr int ConstexprFloorLog2(T v) {
+  // NOTE: not checking is_integral so that this works with Unsigned128
+  static_assert(!std::is_reference_v<T>, "use std::remove_reference_t");
   int rv = 0;
   while (v > T{1}) {
     ++rv;
@@ -74,7 +101,8 @@ constexpr int ConstexprFloorLog2(T v) {
 // Number of low-order zero bits before the first 1 bit. Undefined for 0.
 template <typename T>
 inline int CountTrailingZeroBits(T v) {
-  static_assert(std::is_integral<T>::value, "non-integral type");
+  static_assert(std::is_integral_v<T>, "non-integral type");
+  static_assert(!std::is_reference_v<T>, "use std::remove_reference_t");
   assert(v != 0);
 #ifdef _MSC_VER
   static_assert(sizeof(T) <= sizeof(uint64_t), "type too big");
@@ -115,6 +143,9 @@ namespace detail {
 
 template <typename T>
 int BitsSetToOneFallback(T v) {
+  static_assert(std::is_integral_v<T>, "non-integral type");
+  static_assert(!std::is_reference_v<T>, "use std::remove_reference_t");
+
   const int kBits = static_cast<int>(sizeof(T)) * 8;
   static_assert((kBits & (kBits - 1)) == 0, "must be power of two bits");
   // we static_cast these bit patterns in order to truncate them to the correct
@@ -140,7 +171,9 @@ int BitsSetToOneFallback(T v) {
 // Number of bits set to 1. Also known as "population count".
 template <typename T>
 inline int BitsSetToOne(T v) {
-  static_assert(std::is_integral<T>::value, "non-integral type");
+  static_assert(std::is_integral_v<T>, "non-integral type");
+  static_assert(!std::is_reference_v<T>, "use std::remove_reference_t");
+
 #ifdef _MSC_VER
   static_assert(sizeof(T) <= sizeof(uint64_t), "type too big");
   if (sizeof(T) < sizeof(uint32_t)) {
@@ -192,7 +225,9 @@ inline int BitsSetToOne(T v) {
 
 template <typename T>
 inline int BitParity(T v) {
-  static_assert(std::is_integral<T>::value, "non-integral type");
+  static_assert(std::is_integral_v<T>, "non-integral type");
+  static_assert(!std::is_reference_v<T>, "use std::remove_reference_t");
+
 #ifdef _MSC_VER
   // bit parity == oddness of popcount
   return BitsSetToOne(v) & 1;
@@ -214,7 +249,8 @@ inline int BitParity(T v) {
 // encode/decode big endian.
 template <typename T>
 inline T EndianSwapValue(T v) {
-  static_assert(std::is_integral<T>::value, "non-integral type");
+  static_assert(std::is_integral_v<T>, "non-integral type");
+  static_assert(!std::is_reference_v<T>, "use std::remove_reference_t");
 
 #ifdef _MSC_VER
   if (sizeof(T) == 2) {
@@ -244,6 +280,9 @@ inline T EndianSwapValue(T v) {
 // Reverses the order of bits in an integral value
 template <typename T>
 inline T ReverseBits(T v) {
+  static_assert(std::is_integral_v<T>, "non-integral type");
+  static_assert(!std::is_reference_v<T>, "use std::remove_reference_t");
+
   T r = EndianSwapValue(v);
   const T kHighestByte = T{1} << ((sizeof(T) - 1) * 8);
   const T kEveryByte = kHighestByte | (kHighestByte / 255);
@@ -277,7 +316,8 @@ inline T ReverseBits(T v) {
 // is that all square sub-matrices that include the top row are invertible.
 template <typename T>
 inline T DownwardInvolution(T v) {
-  static_assert(std::is_integral<T>::value, "non-integral type");
+  static_assert(std::is_integral_v<T>, "non-integral type");
+  static_assert(!std::is_reference_v<T>, "use std::remove_reference_t");
   static_assert(sizeof(T) <= 8, "only supported up to 64 bits");
 
   uint64_t r = static_cast<uint64_t>(v);
@@ -296,4 +336,16 @@ inline T DownwardInvolution(T v) {
   return static_cast<T>(r);
 }
 
+// Bitwise-And with typing that allows you to avoid writing an explicit cast
+// to the smaller type, or the type of the right parameter if same size.
+template <typename A, typename B>
+inline std::conditional_t<sizeof(A) < sizeof(B), A, B> BitwiseAnd(A a, B b) {
+  static_assert(std::is_integral_v<A>, "non-integral type");
+  static_assert(std::is_integral_v<B>, "non-integral type");
+  static_assert(!std::is_reference_v<A>, "use std::remove_reference_t");
+  static_assert(!std::is_reference_v<B>, "use std::remove_reference_t");
+  using Smaller = std::conditional_t<sizeof(A) < sizeof(B), A, B>;
+  return static_cast<Smaller>(a & b);
+}
+
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/util/math128.h

@@ -41,13 +41,13 @@ struct Unsigned128 {
     hi = upper;
   }
 
-  explicit operator uint64_t() { return lo; }
-
-  explicit operator uint32_t() { return static_cast<uint32_t>(lo); }
-
-  explicit operator uint16_t() { return static_cast<uint16_t>(lo); }
-
-  explicit operator uint8_t() { return static_cast<uint8_t>(lo); }
+  // Convert to any integer 64 bits or less.
+  template <typename T,
+            typename = std::enable_if_t<std::is_integral_v<T> &&
+                                        sizeof(T) <= sizeof(uint64_t)> >
+  explicit operator T() {
+    return static_cast<T>(lo);
+  }
 };
 
 inline Unsigned128 operator<<(const Unsigned128& lhs, unsigned shift) {
@@ -190,6 +190,16 @@ inline Unsigned128 Multiply64to128(uint64_t a, uint64_t b) {
 #endif
 }
 
+template <>
+inline Unsigned128 BottomNBits(Unsigned128 v, int nbits) {
+  if (nbits < 64) {
+    return BottomNBits(Lower64of128(v), nbits);
+  } else {
+    return (Unsigned128{BottomNBits(Upper64of128(v), nbits - 64)} << 64) |
+           Lower64of128(v);
+  }
+}
+
 template <>
 inline int FloorLog2(Unsigned128 v) {
   if (Upper64of128(v) == 0) {
@@ -236,6 +246,18 @@ inline Unsigned128 DownwardInvolution(Unsigned128 v) {
          DownwardInvolution(Upper64of128(v) ^ Lower64of128(v));
 }
 
+template <typename A>
+inline std::remove_reference_t<A> BitwiseAnd(A a, Unsigned128 b) {
+  static_assert(sizeof(A) <= sizeof(Unsigned128));
+  return static_cast<A>(a & b);
+}
+
+template <typename B>
+inline std::remove_reference_t<B> BitwiseAnd(Unsigned128 a, B b) {
+  static_assert(sizeof(B) <= sizeof(Unsigned128));
+  return static_cast<B>(a & b);
+}
+
 template <typename T>
 struct IsUnsignedUpTo128
     : std::integral_constant<bool, std::is_unsigned<T>::value ||

package/deps/rocksdb/rocksdb/util/mutexlock.h

@@ -12,10 +12,13 @@
 
 #include <atomic>
 #include <functional>
+#include <memory>
 #include <mutex>
 #include <thread>
 
 #include "port/port.h"
+#include "util/fastrange.h"
+#include "util/hash.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -129,10 +132,25 @@ class SpinMutex {
   std::atomic<bool> locked_;
 };
 
-// We want to prevent false sharing
+// For preventing false sharing, especially for mutexes.
+// NOTE: if a mutex is less than half the size of a cache line, it would
+// make more sense for Striped structure below to pack more than one mutex
+// into each cache line, as this would only reduce contention for the same
+// amount of space and cache sharing. However, a mutex is often 40 bytes out
+// of a 64 byte cache line.
 template <class T>
-struct ALIGN_AS(CACHE_LINE_SIZE) LockData {
-  T lock_;
+struct ALIGN_AS(CACHE_LINE_SIZE) CacheAlignedWrapper {
+  T obj_;
+};
+template <class T>
+struct Unwrap {
+  using type = T;
+  static type &Go(T &t) { return t; }
+};
+template <class T>
+struct Unwrap<CacheAlignedWrapper<T>> {
+  using type = T;
+  static type &Go(CacheAlignedWrapper<T> &t) { return t.obj_; }
 };
 
 //
@@ -144,38 +162,28 @@ struct ALIGN_AS(CACHE_LINE_SIZE) LockData {
 // single lock and allowing independent operations to lock different stripes and
 // proceed concurrently, instead of creating contention for a single lock.
 //
-template <class T, class P>
+template <class T, class Key = Slice, class Hash = SliceNPHasher64>
 class Striped {
  public:
-  Striped(size_t stripes, std::function<uint64_t(const P &)> hash)
-      : stripes_(stripes), hash_(hash) {
-    locks_ = reinterpret_cast<LockData<T> *>(
-        port::cacheline_aligned_alloc(sizeof(LockData<T>) * stripes));
-    for (size_t i = 0; i < stripes; i++) {
-      new (&locks_[i]) LockData<T>();
-    }
-  }
+  explicit Striped(size_t stripe_count)
+      : stripe_count_(stripe_count), data_(new T[stripe_count]) {}
 
-  virtual ~Striped() {
-    if (locks_ != nullptr) {
-      assert(stripes_ > 0);
-      for (size_t i = 0; i < stripes_; i++) {
-        locks_[i].~LockData<T>();
-      }
-      port::cacheline_aligned_free(locks_);
-    }
+  using Unwrapped = typename Unwrap<T>::type;
+  Unwrapped &Get(const Key &key, uint64_t seed = 0) {
+    size_t index = FastRangeGeneric(hash_(key, seed), stripe_count_);
+    return Unwrap<T>::Go(data_[index]);
   }
 
-  T *get(const P &key) {
-    uint64_t h = hash_(key);
-    size_t index = h % stripes_;
-    return &reinterpret_cast<LockData<T> *>(&locks_[index])->lock_;
+  size_t ApproximateMemoryUsage() const {
+    // NOTE: could use malloc_usable_size() here, but that could count unmapped
+    // pages and could mess up unit test OccLockBucketsTest::CacheAligned
+    return sizeof(*this) + stripe_count_ * sizeof(T);
   }
 
  private:
-  size_t stripes_;
-  LockData<T> *locks_;
-  std::function<uint64_t(const P &)> hash_;
+  size_t stripe_count_;
+  std::unique_ptr<T[]> data_;
+  Hash hash_;
 };
 
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/util/overload.h

@@ -0,0 +1,23 @@
+//  Copyright (c) Meta Platforms, Inc. and affiliates.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+#pragma once
+
+#include "rocksdb/rocksdb_namespace.h"
+
+namespace ROCKSDB_NAMESPACE {
+
+// A helper template that can combine multiple functors into a single one to be
+// used with std::visit for example. It also works with lambdas, since it
+// comes with an explicit deduction guide.
+template <typename... Ts>
+struct overload : Ts... {
+  using Ts::operator()...;
+};
+
+template <typename... Ts>
+overload(Ts...) -> overload<Ts...>;
+
+}  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/util/rate_limiter.cc

@@ -107,6 +107,23 @@ void GenericRateLimiter::SetBytesPerSecondLocked(int64_t bytes_per_second) {
       std::memory_order_relaxed);
 }
 
+Status GenericRateLimiter::SetSingleBurstBytes(int64_t single_burst_bytes) {
+  if (single_burst_bytes <= 0) {
+    return Status::InvalidArgument(
+        "`single_burst_bytes` must be greater than 0");
+  }
+
+  MutexLock g(&request_mutex_);
+  SetSingleBurstBytesLocked(single_burst_bytes);
+  return Status::OK();
+}
+
+void GenericRateLimiter::SetSingleBurstBytesLocked(int64_t single_burst_bytes) {
+  refill_bytes_per_period_.store(single_burst_bytes, std::memory_order_relaxed);
+  refill_period_us_.store(CalculateRefillPeriodUsLocked(single_burst_bytes),
+                          std::memory_order_relaxed);
+}
+
 void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri,
                                  Statistics* stats) {
   assert(bytes <= refill_bytes_per_period_.load(std::memory_order_relaxed));
@@ -120,7 +137,8 @@ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri,
     static const int kRefillsPerTune = 100;
     std::chrono::microseconds now(NowMicrosMonotonicLocked());
     if (now - tuned_time_ >=
-        kRefillsPerTune * std::chrono::microseconds(refill_period_us_)) {
+        kRefillsPerTune * std::chrono::microseconds(refill_period_us_.load(
+                              std::memory_order_relaxed))) {
       Status s = TuneLocked();
       s.PermitUncheckedError();  //**TODO: What to do on error?
     }
@@ -170,7 +188,7 @@ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri,
         RecordTick(stats, NUMBER_RATE_LIMITER_DRAINS);
         ++num_drains_;
         wait_until_refill_pending_ = true;
-        r.cv.TimedWait(wait_until);
+        clock_->TimedWait(&r.cv, std::chrono::microseconds(wait_until));
         TEST_SYNC_POINT_CALLBACK("GenericRateLimiter::Request:PostTimedWait",
                                  &time_until_refill_us);
         wait_until_refill_pending_ = false;
@@ -179,16 +197,16 @@ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri,
       // Whichever thread reaches here first performs duty (2) as described
       // above.
       RefillBytesAndGrantRequestsLocked();
-      if (r.request_bytes == 0) {
-        // If there is any remaining requests, make sure there exists at least
-        // one candidate is awake for future duties by signaling a front request
-        // of a queue.
-        for (int i = Env::IO_TOTAL - 1; i >= Env::IO_LOW; --i) {
-          std::deque<Req*> queue = queue_[i];
-          if (!queue.empty()) {
-            queue.front()->cv.Signal();
-            break;
-          }
+    }
+    if (r.request_bytes == 0) {
+      // If there is any remaining requests, make sure there exists at least
+      // one candidate is awake for future duties by signaling a front request
+      // of a queue.
+      for (int i = Env::IO_TOTAL - 1; i >= Env::IO_LOW; --i) {
+        auto& queue = queue_[i];
+        if (!queue.empty()) {
+          queue.front()->cv.Signal();
+          break;
         }
       }
     }
@@ -261,7 +279,8 @@ GenericRateLimiter::GeneratePriorityIterationOrderLocked() {
 void GenericRateLimiter::RefillBytesAndGrantRequestsLocked() {
   TEST_SYNC_POINT_CALLBACK(
       "GenericRateLimiter::RefillBytesAndGrantRequestsLocked", &request_mutex_);
-  next_refill_us_ = NowMicrosMonotonicLocked() + refill_period_us_;
+  next_refill_us_ = NowMicrosMonotonicLocked() +
+                    refill_period_us_.load(std::memory_order_relaxed);
   // Carry over the left over quota from the last period
   auto refill_bytes_per_period =
       refill_bytes_per_period_.load(std::memory_order_relaxed);
@@ -299,13 +318,28 @@ void GenericRateLimiter::RefillBytesAndGrantRequestsLocked() {
 
 int64_t GenericRateLimiter::CalculateRefillBytesPerPeriodLocked(
     int64_t rate_bytes_per_sec) {
+  int64_t refill_period_us = refill_period_us_.load(std::memory_order_relaxed);
   if (std::numeric_limits<int64_t>::max() / rate_bytes_per_sec <
-      refill_period_us_) {
+      refill_period_us) {
+    // Avoid unexpected result in the overflow case. The result now is still
+    // inaccurate but is a number that is large enough.
+    return std::numeric_limits<int64_t>::max() / kMicrosecondsPerSecond;
+  } else {
+    return rate_bytes_per_sec * refill_period_us / kMicrosecondsPerSecond;
+  }
+}
+
+int64_t GenericRateLimiter::CalculateRefillPeriodUsLocked(
+    int64_t single_burst_bytes) {
+  int64_t rate_bytes_per_sec =
+      rate_bytes_per_sec_.load(std::memory_order_relaxed);
+  if (std::numeric_limits<int64_t>::max() / single_burst_bytes <
+      kMicrosecondsPerSecond) {
     // Avoid unexpected result in the overflow case. The result now is still
     // inaccurate but is a number that is large enough.
-    return std::numeric_limits<int64_t>::max() / 1000000;
+    return std::numeric_limits<int64_t>::max() / rate_bytes_per_sec;
   } else {
-    return rate_bytes_per_sec * refill_period_us_ / 1000000;
+    return single_burst_bytes * kMicrosecondsPerSecond / rate_bytes_per_sec;
   }
 }
 
@@ -320,10 +354,11 @@ Status GenericRateLimiter::TuneLocked() {
   std::chrono::microseconds prev_tuned_time = tuned_time_;
   tuned_time_ = std::chrono::microseconds(NowMicrosMonotonicLocked());
 
+  int64_t refill_period_us = refill_period_us_.load(std::memory_order_relaxed);
   int64_t elapsed_intervals = (tuned_time_ - prev_tuned_time +
-                               std::chrono::microseconds(refill_period_us_) -
+                               std::chrono::microseconds(refill_period_us) -
                                std::chrono::microseconds(1)) /
-                              std::chrono::microseconds(refill_period_us_);
+                              std::chrono::microseconds(refill_period_us);
   // We tune every kRefillsPerTune intervals, so the overflow and division-by-
   // zero conditions should never happen.
   assert(num_drains_ <= std::numeric_limits<int64_t>::max() / 100);

package/deps/rocksdb/rocksdb/util/rate_limiter_impl.h

@@ -36,6 +36,8 @@ class GenericRateLimiter : public RateLimiter {
   // This API allows user to dynamically change rate limiter's bytes per second.
   virtual void SetBytesPerSecond(int64_t bytes_per_second) override;
 
+  virtual Status SetSingleBurstBytes(int64_t single_burst_bytes) override;
+
   // Request for token to write bytes. If this request can not be satisfied,
   // the call is blocked. Caller is responsible to make sure
   // bytes <= GetSingleBurstBytes() and bytes >= 0. Negative bytes
@@ -102,11 +104,14 @@ class GenericRateLimiter : public RateLimiter {
   }
 
  private:
+  static constexpr int kMicrosecondsPerSecond = 1000000;
   void RefillBytesAndGrantRequestsLocked();
   std::vector<Env::IOPriority> GeneratePriorityIterationOrderLocked();
   int64_t CalculateRefillBytesPerPeriodLocked(int64_t rate_bytes_per_sec);
+  int64_t CalculateRefillPeriodUsLocked(int64_t single_burst_bytes);
   Status TuneLocked();
   void SetBytesPerSecondLocked(int64_t bytes_per_second);
+  void SetSingleBurstBytesLocked(int64_t single_burst_bytes);
 
   uint64_t NowMicrosMonotonicLocked() {
     return clock_->NowNanos() / std::milli::den;
@@ -115,7 +120,7 @@ class GenericRateLimiter : public RateLimiter {
   // This mutex guard all internal states
   mutable port::Mutex request_mutex_;
 
-  const int64_t refill_period_us_;
+  std::atomic<int64_t> refill_period_us_;
 
   std::atomic<int64_t> rate_bytes_per_sec_;
   std::atomic<int64_t> refill_bytes_per_period_;

package/deps/rocksdb/rocksdb/util/rate_limiter_test.cc

@@ -15,6 +15,7 @@
 #include "db/db_test_util.h"
 #include "port/port.h"
 #include "rocksdb/system_clock.h"
+#include "test_util/mock_time_env.h"
 #include "test_util/sync_point.h"
 #include "test_util/testharness.h"
 #include "util/random.h"
@@ -464,31 +465,21 @@ TEST_F(RateLimiterTest, AutoTuneIncreaseWhenFull) {
   const std::chrono::seconds kTimePerRefill(1);
   const int kRefillsPerTune = 100;  // needs to match util/rate_limiter.cc
 
-  SpecialEnv special_env(Env::Default(), /*time_elapse_only_sleep*/ true);
+  auto mock_clock =
+      std::make_shared<MockSystemClock>(Env::Default()->GetSystemClock());
 
   auto stats = CreateDBStatistics();
   std::unique_ptr<RateLimiter> rate_limiter(new GenericRateLimiter(
       1000 /* rate_bytes_per_sec */,
       std::chrono::microseconds(kTimePerRefill).count(), 10 /* fairness */,
-      RateLimiter::Mode::kWritesOnly, special_env.GetSystemClock(),
-      true /* auto_tuned */));
-
-  // Rate limiter uses `CondVar::TimedWait()`, which does not have access to the
-  // `Env` to advance its time according to the fake wait duration. The
-  // workaround is to install a callback that advance the `Env`'s mock time.
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->SetCallBack(
-      "GenericRateLimiter::Request:PostTimedWait", [&](void* arg) {
-        int64_t time_waited_us = *static_cast<int64_t*>(arg);
-        special_env.SleepForMicroseconds(static_cast<int>(time_waited_us));
-      });
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+      RateLimiter::Mode::kWritesOnly, mock_clock, true /* auto_tuned */));
 
   // verify rate limit increases after a sequence of periods where rate limiter
   // is always drained
   int64_t orig_bytes_per_sec = rate_limiter->GetSingleBurstBytes();
   rate_limiter->Request(orig_bytes_per_sec, Env::IO_HIGH, stats.get(),
                         RateLimiter::OpType::kWrite);
-  while (std::chrono::microseconds(special_env.NowMicros()) <=
+  while (std::chrono::microseconds(mock_clock->NowMicros()) <=
          kRefillsPerTune * kTimePerRefill) {
     rate_limiter->Request(orig_bytes_per_sec, Env::IO_HIGH, stats.get(),
                           RateLimiter::OpType::kWrite);
@@ -496,13 +487,9 @@ TEST_F(RateLimiterTest, AutoTuneIncreaseWhenFull) {
   int64_t new_bytes_per_sec = rate_limiter->GetSingleBurstBytes();
   ASSERT_GT(new_bytes_per_sec, orig_bytes_per_sec);
 
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearCallBack(
-      "GenericRateLimiter::Request:PostTimedWait");
-
   // decreases after a sequence of periods where rate limiter is not drained
   orig_bytes_per_sec = new_bytes_per_sec;
-  special_env.SleepForMicroseconds(static_cast<int>(
+  mock_clock->SleepForMicroseconds(static_cast<int>(
       kRefillsPerTune * std::chrono::microseconds(kTimePerRefill).count()));
   // make a request so tuner can be triggered
   rate_limiter->Request(1 /* bytes */, Env::IO_HIGH, stats.get(),
@@ -511,6 +498,90 @@ TEST_F(RateLimiterTest, AutoTuneIncreaseWhenFull) {
   ASSERT_LT(new_bytes_per_sec, orig_bytes_per_sec);
 }
 
+TEST_F(RateLimiterTest, WaitHangingBug) {
+  // At t=0: Threads 0 and 1 request `kBytesPerRefill` bytes at low-pri. One
+  // will be granted immediately and the other will enter `TimedWait()`.
+  //
+  // At t=`kMicrosPerRefill`: Thread 2 requests `kBytesPerRefill` bytes at
+  // low-pri. Thread 2's request enters the queue. To expose the bug scenario,
+  // `SyncPoint`s ensure this happens while the lock is temporarily released in
+  // `TimedWait()`. Before the bug fix, Thread 2's request would then hang in
+  // `Wait()` interminably.
+  const int kBytesPerSecond = 100;
+  const int kMicrosPerSecond = 1000 * 1000;
+  const int kMicrosPerRefill = kMicrosPerSecond;
+  const int kBytesPerRefill =
+      kBytesPerSecond * kMicrosPerRefill / kMicrosPerSecond;
+
+  auto mock_clock =
+      std::make_shared<MockSystemClock>(Env::Default()->GetSystemClock());
+  std::unique_ptr<RateLimiter> limiter(new GenericRateLimiter(
+      kBytesPerSecond, kMicrosPerRefill, 10 /* fairness */,
+      RateLimiter::Mode::kWritesOnly, mock_clock, false /* auto_tuned */));
+  std::array<std::thread, 3> request_threads;
+
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
+      {{"RateLimiterTest::WaitHangingBug:InitialRequestsReady",
+        "MockSystemClock::TimedWait:UnlockedPreSleep"},
+       {"MockSystemClock::TimedWait:UnlockedPostSleep1",
+        "RateLimiterTest::WaitHangingBug:TestThreadRequestBegin"},
+       {"RateLimiterTest::WaitHangingBug:TestThreadRequestEnd",
+        "MockSystemClock::TimedWait:UnlockedPostSleep2"}});
+  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
+
+  for (int i = 0; i < 2; i++) {
+    request_threads[i] = std::thread([&]() {
+      limiter->Request(kBytesPerRefill /* bytes */, Env::IOPriority::IO_LOW,
+                       nullptr /* stats */, RateLimiter::OpType::kWrite);
+    });
+  }
+  while (limiter->GetTotalRequests() < 2) {
+  }
+  TEST_SYNC_POINT("RateLimiterTest::WaitHangingBug:InitialRequestsReady");
+
+  TEST_SYNC_POINT("RateLimiterTest::WaitHangingBug:TestThreadRequestBegin");
+  request_threads[2] = std::thread([&]() {
+    limiter->Request(kBytesPerRefill /* bytes */, Env::IOPriority::IO_LOW,
+                     nullptr /* stats */, RateLimiter::OpType::kWrite);
+  });
+  while (limiter->GetTotalRequests() < 3) {
+  }
+  TEST_SYNC_POINT("RateLimiterTest::WaitHangingBug:TestThreadRequestEnd");
+
+  for (int i = 0; i < 3; i++) {
+    request_threads[i].join();
+  }
+}
+
+TEST_F(RateLimiterTest, RuntimeSingleBurstBytesChange) {
+  constexpr int kMicrosecondsPerSecond = 1000000;
+
+  const int64_t kRateBytesPerSec = 400;
+
+  const int64_t kOldSingleBurstBytes = 100;
+  const int64_t kOldRefillPeriodUs =
+      kOldSingleBurstBytes * kMicrosecondsPerSecond / kRateBytesPerSec;
+  const int64_t kNewSingleBurstBytes = kOldSingleBurstBytes * 2;
+
+  SpecialEnv special_env(Env::Default(), /*time_elapse_only_sleep*/ true);
+  std::unique_ptr<RateLimiter> limiter(new GenericRateLimiter(
+      kRateBytesPerSec, kOldRefillPeriodUs, 10 /* fairness */,
+      RateLimiter::Mode::kWritesOnly, special_env.GetSystemClock(),
+      false /* auto_tuned */));
+
+  ASSERT_EQ(kOldSingleBurstBytes, limiter->GetSingleBurstBytes());
+
+  ASSERT_TRUE(limiter->SetSingleBurstBytes(0).IsInvalidArgument());
+  ASSERT_OK(limiter->SetSingleBurstBytes(kNewSingleBurstBytes));
+  ASSERT_EQ(kNewSingleBurstBytes, limiter->GetSingleBurstBytes());
+
+  // If the updated single burst bytes is not reflected in the bytes
+  // granting process, this request will hang forever.
+  limiter->Request(limiter->GetSingleBurstBytes() /* bytes */,
+                   Env::IOPriority::IO_USER, nullptr /* stats */,
+                   RateLimiter::OpType::kWrite);
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

package/deps/rocksdb/rocksdb/util/single_thread_executor.h

@@ -8,6 +8,7 @@
 
 #if USE_COROUTINES
 #include <atomic>
+#include <queue>
 
 #include "folly/CPortability.h"
 #include "folly/CppAttributes.h"

package/deps/rocksdb/rocksdb/util/slice_test.cc

@@ -243,6 +243,36 @@ TEST_F(SmallEnumSetTest, SmallEnumSetTest2) {
   }
 }
 
+// ***************************************************************** //
+// Unit test for Status
+TEST(StatusTest, Update) {
+  const Status ok = Status::OK();
+  const Status inc = Status::Incomplete("blah");
+  const Status notf = Status::NotFound("meow");
+
+  Status s = ok;
+  ASSERT_TRUE(s.UpdateIfOk(Status::Corruption("bad")).IsCorruption());
+  ASSERT_TRUE(s.IsCorruption());
+
+  s = ok;
+  ASSERT_TRUE(s.UpdateIfOk(Status::OK()).ok());
+  ASSERT_TRUE(s.UpdateIfOk(ok).ok());
+  ASSERT_TRUE(s.ok());
+
+  ASSERT_TRUE(s.UpdateIfOk(inc).IsIncomplete());
+  ASSERT_TRUE(s.IsIncomplete());
+
+  ASSERT_TRUE(s.UpdateIfOk(notf).IsIncomplete());
+  ASSERT_TRUE(s.UpdateIfOk(ok).IsIncomplete());
+  ASSERT_TRUE(s.IsIncomplete());
+
+  // Keeps left-most non-OK status
+  s = ok;
+  ASSERT_TRUE(
+      s.UpdateIfOk(Status()).UpdateIfOk(notf).UpdateIfOk(inc).IsNotFound());
+  ASSERT_TRUE(s.IsNotFound());
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {