@nxtedition/rocksdb 5.2.21 → 5.2.26

package/deps/rocksdb/rocksdb/util/repeatable_thread_test.cc

@@ -3,44 +3,46 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
+#include "util/repeatable_thread.h"
+
 #include <atomic>
 #include <memory>
 
 #include "db/db_test_util.h"
+#include "test_util/mock_time_env.h"
 #include "test_util/sync_point.h"
 #include "test_util/testharness.h"
-#include "util/repeatable_thread.h"
 
 class RepeatableThreadTest : public testing::Test {
  public:
   RepeatableThreadTest()
-      : mock_env_(new ROCKSDB_NAMESPACE::MockTimeEnv(
-            ROCKSDB_NAMESPACE::Env::Default())) {}
+      : mock_clock_(std::make_shared<ROCKSDB_NAMESPACE::MockSystemClock>(
+            ROCKSDB_NAMESPACE::SystemClock::Default())) {}
 
  protected:
-  std::unique_ptr<ROCKSDB_NAMESPACE::MockTimeEnv> mock_env_;
+  std::shared_ptr<ROCKSDB_NAMESPACE::MockSystemClock> mock_clock_;
 };
 
 TEST_F(RepeatableThreadTest, TimedTest) {
   constexpr uint64_t kSecond = 1000000;  // 1s = 1000000us
   constexpr int kIteration = 3;
-  ROCKSDB_NAMESPACE::Env* env = ROCKSDB_NAMESPACE::Env::Default();
+  const auto& clock = ROCKSDB_NAMESPACE::SystemClock::Default();
   ROCKSDB_NAMESPACE::port::Mutex mutex;
   ROCKSDB_NAMESPACE::port::CondVar test_cv(&mutex);
   int count = 0;
-  uint64_t prev_time = env->NowMicros();
+  uint64_t prev_time = clock->NowMicros();
   ROCKSDB_NAMESPACE::RepeatableThread thread(
       [&] {
         ROCKSDB_NAMESPACE::MutexLock l(&mutex);
         count++;
-        uint64_t now = env->NowMicros();
+        uint64_t now = clock->NowMicros();
         assert(count == 1 || prev_time + 1 * kSecond <= now);
         prev_time = now;
         if (count >= kIteration) {
           test_cv.SignalAll();
         }
       },
-      "rt_test", env, 1 * kSecond);
+      "rt_test", clock.get(), 1 * kSecond);
   // Wait for execution finish.
   {
     ROCKSDB_NAMESPACE::MutexLock l(&mutex);
@@ -56,7 +58,7 @@ TEST_F(RepeatableThreadTest, TimedTest) {
 TEST_F(RepeatableThreadTest, MockEnvTest) {
   constexpr uint64_t kSecond = 1000000;  // 1s = 1000000us
   constexpr int kIteration = 3;
-  mock_env_->set_current_time(0);  // in seconds
+  mock_clock_->SetCurrentTime(0);  // in seconds
   std::atomic<int> count{0};
 
 #if defined(OS_MACOSX) && !defined(NDEBUG)
@@ -72,26 +74,27 @@ TEST_F(RepeatableThreadTest, MockEnvTest) {
         // immediately, the mutex will not be released, and
         // RepeatableThread::TEST_WaitForRun never has a chance to execute the
         // callback which, in this case, updates the result returned by
-        // mock_env->NowMicros. Consequently, RepeatableThread::wait cannot
+        // mock_clock->NowMicros. Consequently, RepeatableThread::wait cannot
         // break out of the loop, causing test to hang. The extra 1000 seconds
         // is a best-effort approach because there seems no reliable and
         // deterministic way to provide the aforementioned guarantee. By the
         // time RepeatableThread::wait is called, it is no guarantee that the
-        // delay + mock_env->NowMicros will be greater than the current real
+        // delay + mock_clock->NowMicros will be greater than the current real
         // time. However, 1000 seconds should be sufficient in most cases.
         uint64_t time_us = *reinterpret_cast<uint64_t*>(arg);
-        if (time_us < mock_env_->RealNowMicros()) {
-          *reinterpret_cast<uint64_t*>(arg) = mock_env_->RealNowMicros() + 1000;
+        if (time_us < mock_clock_->RealNowMicros()) {
+          *reinterpret_cast<uint64_t*>(arg) =
+              mock_clock_->RealNowMicros() + 1000;
         }
       });
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
 #endif  // OS_MACOSX && !NDEBUG
 
   ROCKSDB_NAMESPACE::RepeatableThread thread(
-      [&] { count++; }, "rt_test", mock_env_.get(), 1 * kSecond, 1 * kSecond);
+      [&] { count++; }, "rt_test", mock_clock_.get(), 1 * kSecond, 1 * kSecond);
   for (int i = 1; i <= kIteration; i++) {
     // Bump current time
-    thread.TEST_WaitForRun([&] { mock_env_->set_current_time(i); });
+    thread.TEST_WaitForRun([&] { mock_clock_->SetCurrentTime(i); });
   }
   // Test function should be exectued exactly kIteraion times.
   ASSERT_EQ(kIteration, count.load());

package/deps/rocksdb/rocksdb/util/ribbon_alg.h

@@ -8,6 +8,7 @@
 #include <array>
 #include <memory>
 
+#include "rocksdb/rocksdb_namespace.h"
 #include "util/math128.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -501,12 +502,13 @@ namespace ribbon {
 //   // slot index i.
 //   void Prefetch(Index i) const;
 //
-//   // Returns a pointer to CoeffRow for slot index i.
-//   CoeffRow* CoeffRowPtr(Index i);
-//
-//   // Returns a pointer to ResultRow for slot index i. (Gaussian row
-//   // operations involve both side of the equation.)
-//   ResultRow* ResultRowPtr(Index i);
+//   // Load or store CoeffRow and ResultRow for slot index i.
+//   // (Gaussian row operations involve both sides of the equation.)
+//   // Bool `for_back_subst` indicates that customizing values for
+//   // unconstrained solution rows (cr == 0) is allowed.
+//   void LoadRow(Index i, CoeffRow *cr, ResultRow *rr, bool for_back_subst)
+//        const;
+//   void StoreRow(Index i, CoeffRow cr, ResultRow rr);
 //
 //   // Returns the number of columns that can start an r-sequence of
 //   // coefficients, which is the number of slots minus r (kCoeffBits)
@@ -548,6 +550,7 @@ bool BandingAdd(BandingStorage *bs, typename BandingStorage::Index start,
                 typename BandingStorage::CoeffRow cr, BacktrackStorage *bts,
                 typename BandingStorage::Index *backtrack_pos) {
   using CoeffRow = typename BandingStorage::CoeffRow;
+  using ResultRow = typename BandingStorage::ResultRow;
   using Index = typename BandingStorage::Index;
 
   Index i = start;
@@ -561,18 +564,19 @@ bool BandingAdd(BandingStorage *bs, typename BandingStorage::Index start,
 
   for (;;) {
     assert((cr & 1) == 1);
-    CoeffRow other = *(bs->CoeffRowPtr(i));
-    if (other == 0) {
-      *(bs->CoeffRowPtr(i)) = cr;
-      *(bs->ResultRowPtr(i)) = rr;
+    CoeffRow cr_at_i;
+    ResultRow rr_at_i;
+    bs->LoadRow(i, &cr_at_i, &rr_at_i, /* for_back_subst */ false);
+    if (cr_at_i == 0) {
+      bs->StoreRow(i, cr, rr);
       bts->BacktrackPut(*backtrack_pos, i);
       ++*backtrack_pos;
       return true;
     }
-    assert((other & 1) == 1);
+    assert((cr_at_i & 1) == 1);
     // Gaussian row reduction
-    cr ^= other;
-    rr ^= *(bs->ResultRowPtr(i));
+    cr ^= cr_at_i;
+    rr ^= rr_at_i;
     if (cr == 0) {
       // Inconsistency or (less likely) redundancy
       break;
@@ -678,12 +682,11 @@ bool BandingAddRange(BandingStorage *bs, BacktrackStorage *bts,
     while (backtrack_pos > 0) {
       --backtrack_pos;
       Index i = bts->BacktrackGet(backtrack_pos);
-      *(bs->CoeffRowPtr(i)) = 0;
-      // Not strictly required, but is required for good FP rate on
-      // inputs that might have been backtracked out. (We don't want
-      // anything we've backtracked on to leak into final result, as
-      // that might not be "harmless".)
-      *(bs->ResultRowPtr(i)) = 0;
+      // Clearing the ResultRow is not strictly required, but is required
+      // for good FP rate on inputs that might have been backtracked out.
+      // (We don't want anything we've backtracked on to leak into final
+      // result, as that might not be "harmless".)
+      bs->StoreRow(i, 0, 0);
     }
   }
   return false;
@@ -780,8 +783,9 @@ void SimpleBackSubst(SimpleSolutionStorage *sss, const BandingStorage &bs) {
 
   for (Index i = num_slots; i > 0;) {
     --i;
-    CoeffRow cr = *const_cast<BandingStorage &>(bs).CoeffRowPtr(i);
-    ResultRow rr = *const_cast<BandingStorage &>(bs).ResultRowPtr(i);
+    CoeffRow cr;
+    ResultRow rr;
+    bs.LoadRow(i, &cr, &rr, /* for_back_subst */ true);
     // solution row
     ResultRow sr = 0;
     for (Index j = 0; j < kResultBits; ++j) {
@@ -976,8 +980,9 @@ inline void BackSubstBlock(typename BandingStorage::CoeffRow *state,
 
   for (Index i = start_slot + kCoeffBits; i > start_slot;) {
     --i;
-    CoeffRow cr = *const_cast<BandingStorage &>(bs).CoeffRowPtr(i);
-    ResultRow rr = *const_cast<BandingStorage &>(bs).ResultRowPtr(i);
+    CoeffRow cr;
+    ResultRow rr;
+    bs.LoadRow(i, &cr, &rr, /* for_back_subst */ true);
     for (Index j = 0; j < num_columns; ++j) {
       // Compute next solution bit at row i, column j (see derivation below)
       CoeffRow tmp = state[j] << 1;
@@ -1050,13 +1055,13 @@ void InterleavedBackSubst(InterleavedSolutionStorage *iss,
   std::unique_ptr<CoeffRow[]> state{new CoeffRow[num_columns]()};
 
   Index block = num_blocks;
-  Index segment = num_segments;
+  Index segment_num = num_segments;
   while (block > upper_start_block) {
     --block;
     BackSubstBlock(state.get(), num_columns, bs, block * kCoeffBits);
-    segment -= num_columns;
+    segment_num -= num_columns;
     for (Index i = 0; i < num_columns; ++i) {
-      iss->StoreSegment(segment + i, state[i]);
+      iss->StoreSegment(segment_num + i, state[i]);
     }
   }
   // Now (if applicable), region using lower number of columns
@@ -1066,60 +1071,92 @@ void InterleavedBackSubst(InterleavedSolutionStorage *iss,
   while (block > 0) {
     --block;
     BackSubstBlock(state.get(), num_columns, bs, block * kCoeffBits);
-    segment -= num_columns;
+    segment_num -= num_columns;
     for (Index i = 0; i < num_columns; ++i) {
-      iss->StoreSegment(segment + i, state[i]);
+      iss->StoreSegment(segment_num + i, state[i]);
     }
   }
   // Verify everything processed
   assert(block == 0);
-  assert(segment == 0);
+  assert(segment_num == 0);
 }
 
-// General PHSF query a key from InterleavedSolutionStorage.
+// Prefetch memory for a key in InterleavedSolutionStorage.
 template <typename InterleavedSolutionStorage, typename PhsfQueryHasher>
-typename InterleavedSolutionStorage::ResultRow InterleavedPhsfQuery(
+inline void InterleavedPrepareQuery(
     const typename PhsfQueryHasher::Key &key, const PhsfQueryHasher &hasher,
-    const InterleavedSolutionStorage &iss) {
+    const InterleavedSolutionStorage &iss,
+    typename PhsfQueryHasher::Hash *saved_hash,
+    typename InterleavedSolutionStorage::Index *saved_segment_num,
+    typename InterleavedSolutionStorage::Index *saved_num_columns,
+    typename InterleavedSolutionStorage::Index *saved_start_bit) {
   using Hash = typename PhsfQueryHasher::Hash;
-
   using CoeffRow = typename InterleavedSolutionStorage::CoeffRow;
   using Index = typename InterleavedSolutionStorage::Index;
-  using ResultRow = typename InterleavedSolutionStorage::ResultRow;
 
   static_assert(sizeof(Index) == sizeof(typename PhsfQueryHasher::Index),
                 "must be same");
-  static_assert(sizeof(CoeffRow) == sizeof(typename PhsfQueryHasher::CoeffRow),
-                "must be same");
-
-  constexpr auto kCoeffBits = static_cast<Index>(sizeof(CoeffRow) * 8U);
 
   const Hash hash = hasher.GetHash(key);
   const Index start_slot = hasher.GetStart(hash, iss.GetNumStarts());
 
+  constexpr auto kCoeffBits = static_cast<Index>(sizeof(CoeffRow) * 8U);
+
   const Index upper_start_block = iss.GetUpperStartBlock();
   Index num_columns = iss.GetUpperNumColumns();
   Index start_block_num = start_slot / kCoeffBits;
-  Index segment = start_block_num * num_columns -
-                  std::min(start_block_num, upper_start_block);
+  Index segment_num = start_block_num * num_columns -
+                      std::min(start_block_num, upper_start_block);
   // Change to lower num columns if applicable.
   // (This should not compile to a conditional branch.)
   num_columns -= (start_block_num < upper_start_block) ? 1 : 0;
 
-  const CoeffRow cr = hasher.GetCoeffRow(hash);
   Index start_bit = start_slot % kCoeffBits;
 
+  Index segment_count = num_columns + (start_bit == 0 ? 0 : num_columns);
+
+  iss.PrefetchSegmentRange(segment_num, segment_num + segment_count);
+
+  *saved_hash = hash;
+  *saved_segment_num = segment_num;
+  *saved_num_columns = num_columns;
+  *saved_start_bit = start_bit;
+}
+
+// General PHSF query from InterleavedSolutionStorage, using data for
+// the query key from InterleavedPrepareQuery
+template <typename InterleavedSolutionStorage, typename PhsfQueryHasher>
+inline typename InterleavedSolutionStorage::ResultRow InterleavedPhsfQuery(
+    typename PhsfQueryHasher::Hash hash,
+    typename InterleavedSolutionStorage::Index segment_num,
+    typename InterleavedSolutionStorage::Index num_columns,
+    typename InterleavedSolutionStorage::Index start_bit,
+    const PhsfQueryHasher &hasher, const InterleavedSolutionStorage &iss) {
+  using CoeffRow = typename InterleavedSolutionStorage::CoeffRow;
+  using Index = typename InterleavedSolutionStorage::Index;
+  using ResultRow = typename InterleavedSolutionStorage::ResultRow;
+
+  static_assert(sizeof(Index) == sizeof(typename PhsfQueryHasher::Index),
+                "must be same");
+  static_assert(sizeof(CoeffRow) == sizeof(typename PhsfQueryHasher::CoeffRow),
+                "must be same");
+
+  constexpr auto kCoeffBits = static_cast<Index>(sizeof(CoeffRow) * 8U);
+
+  const CoeffRow cr = hasher.GetCoeffRow(hash);
+
   ResultRow sr = 0;
-  const CoeffRow cr_left = cr << start_bit;
+  const CoeffRow cr_left = cr << static_cast<unsigned>(start_bit);
   for (Index i = 0; i < num_columns; ++i) {
-    sr ^= BitParity(iss.LoadSegment(segment + i) & cr_left) << i;
+    sr ^= BitParity(iss.LoadSegment(segment_num + i) & cr_left) << i;
   }
 
   if (start_bit > 0) {
-    segment += num_columns;
-    const CoeffRow cr_right = cr >> (kCoeffBits - start_bit);
+    segment_num += num_columns;
+    const CoeffRow cr_right =
+        cr >> static_cast<unsigned>(kCoeffBits - start_bit);
     for (Index i = 0; i < num_columns; ++i) {
-      sr ^= BitParity(iss.LoadSegment(segment + i) & cr_right) << i;
+      sr ^= BitParity(iss.LoadSegment(segment_num + i) & cr_right) << i;
     }
   }
 
@@ -1128,12 +1165,12 @@ typename InterleavedSolutionStorage::ResultRow InterleavedPhsfQuery(
 
 // Filter query a key from InterleavedFilterQuery.
 template <typename InterleavedSolutionStorage, typename FilterQueryHasher>
-bool InterleavedFilterQuery(const typename FilterQueryHasher::Key &key,
-                            const FilterQueryHasher &hasher,
-                            const InterleavedSolutionStorage &iss) {
-  // BEGIN mostly copied from InterleavedPhsfQuery
-  using Hash = typename FilterQueryHasher::Hash;
-
+inline bool InterleavedFilterQuery(
+    typename FilterQueryHasher::Hash hash,
+    typename InterleavedSolutionStorage::Index segment_num,
+    typename InterleavedSolutionStorage::Index num_columns,
+    typename InterleavedSolutionStorage::Index start_bit,
+    const FilterQueryHasher &hasher, const InterleavedSolutionStorage &iss) {
   using CoeffRow = typename InterleavedSolutionStorage::CoeffRow;
   using Index = typename InterleavedSolutionStorage::Index;
   using ResultRow = typename InterleavedSolutionStorage::ResultRow;
@@ -1149,41 +1186,28 @@ bool InterleavedFilterQuery(const typename FilterQueryHasher::Key &key,
 
   constexpr auto kCoeffBits = static_cast<Index>(sizeof(CoeffRow) * 8U);
 
-  const Hash hash = hasher.GetHash(key);
-  const Index start_slot = hasher.GetStart(hash, iss.GetNumStarts());
-
-  const Index upper_start_block = iss.GetUpperStartBlock();
-  Index num_columns = iss.GetUpperNumColumns();
-  Index start_block_num = start_slot / kCoeffBits;
-  Index segment = start_block_num * num_columns -
-                  std::min(start_block_num, upper_start_block);
-  // Change to lower num columns if applicable.
-  // (This should not compile to a conditional branch.)
-  num_columns -= (start_block_num < upper_start_block) ? 1 : 0;
-
   const CoeffRow cr = hasher.GetCoeffRow(hash);
-  Index start_bit = start_slot % kCoeffBits;
-  // END mostly copied from InterleavedPhsfQuery.
-
   const ResultRow expected = hasher.GetResultRowFromHash(hash);
 
   // TODO: consider optimizations such as
-  // * mask fetched values and shift cr, rather than shifting fetched values
   // * get rid of start_bit == 0 condition with careful fetching & shifting
   if (start_bit == 0) {
     for (Index i = 0; i < num_columns; ++i) {
-      if (BitParity(iss.LoadSegment(segment + i) & cr) !=
+      if (BitParity(iss.LoadSegment(segment_num + i) & cr) !=
           (static_cast<int>(expected >> i) & 1)) {
         return false;
       }
     }
   } else {
+    const CoeffRow cr_left = cr << static_cast<unsigned>(start_bit);
+    const CoeffRow cr_right =
+        cr >> static_cast<unsigned>(kCoeffBits - start_bit);
+
     for (Index i = 0; i < num_columns; ++i) {
-      CoeffRow soln_col =
-          (iss.LoadSegment(segment + i) >> static_cast<unsigned>(start_bit)) |
-          (iss.LoadSegment(segment + num_columns + i)
-           << static_cast<unsigned>(kCoeffBits - start_bit));
-      if (BitParity(soln_col & cr) != (static_cast<int>(expected >> i) & 1)) {
+      CoeffRow soln_data =
+          (iss.LoadSegment(segment_num + i) & cr_left) ^
+          (iss.LoadSegment(segment_num + num_columns + i) & cr_right);
+      if (BitParity(soln_data) != (static_cast<int>(expected >> i) & 1)) {
         return false;
       }
     }