@nxtedition/rocksdb 6.0.2 → 7.0.0-alpha.2

package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h

@@ -89,6 +89,7 @@ class FilePrefetchBuffer {
     // while curr_ is being consumed. If data is overlapping in two buffers,
     // data is copied to third buffer to return continuous buffer.
     bufs_.resize(3);
+    (void)async_io_;
   }
 
   ~FilePrefetchBuffer() {
@@ -131,10 +132,21 @@ class FilePrefetchBuffer {
                   uint64_t offset, size_t n,
                   Env::IOPriority rate_limiter_priority);
 
+  // Request for reading the data from a file asynchronously.
+  // If data already exists in the buffer, result will be updated.
+  // reader                : the file reader.
+  // offset                : the file offset to start reading from.
+  // n                     : the number of bytes to read.
+  // rate_limiter_priority : rate limiting priority, or `Env::IO_TOTAL` to
+  //                         bypass.
+  // result                : if data already exists in the buffer, result will
+  //                         be updated with the data.
+  //
+  // If data already exist in the buffer, it will return Status::OK, otherwise
+  // it will send asynchronous request and return Status::TryAgain.
   Status PrefetchAsync(const IOOptions& opts, RandomAccessFileReader* reader,
-                       uint64_t offset, size_t length, size_t readahead_size,
-                       Env::IOPriority rate_limiter_priority,
-                       bool& copy_to_third_buffer);
+                       uint64_t offset, size_t n,
+                       Env::IOPriority rate_limiter_priority, Slice* result);
 
   // Tries returning the data for a file read from this buffer if that data is
   // in the buffer.
@@ -159,8 +171,7 @@ class FilePrefetchBuffer {
   bool TryReadFromCacheAsync(const IOOptions& opts,
                              RandomAccessFileReader* reader, uint64_t offset,
                              size_t n, Slice* result, Status* status,
-                             Env::IOPriority rate_limiter_priority,
-                             bool for_compaction /* = false */);
+                             Env::IOPriority rate_limiter_priority);
 
   // The minimum `offset` ever passed to TryReadFromCache(). This will nly be
   // tracked if track_min_offset = true.
@@ -207,22 +218,6 @@ class FilePrefetchBuffer {
     }
   }
 
-  bool IsEligibleForPrefetch(uint64_t offset, size_t n) {
-    // Prefetch only if this read is sequential otherwise reset readahead_size_
-    // to initial value.
-    if (!IsBlockSequential(offset)) {
-      UpdateReadPattern(offset, n, false /*decrease_readaheadsize*/);
-      ResetValues();
-      return false;
-    }
-    num_file_reads_++;
-    if (num_file_reads_ <= kMinNumFileReadsToStartAutoReadahead) {
-      UpdateReadPattern(offset, n, false /*decrease_readaheadsize*/);
-      return false;
-    }
-    return true;
-  }
-
   // Callback function passed to underlying FS in case of asynchronous reads.
   void PrefetchAsyncCallback(const FSReadRequest& req, void* cb_arg);
 
@@ -234,6 +229,17 @@ class FilePrefetchBuffer {
                              size_t roundup_len, size_t index, bool refit_tail,
                              uint64_t& chunk_len);
 
+  // It calls Poll API if any there is any pending asynchronous request. It then
+  // checks if data is in any buffer. It clears the outdated data and swaps the
+  // buffers if required.
+  void PollAndUpdateBuffersIfNeeded(uint64_t offset);
+
+  Status PrefetchAsyncInternal(const IOOptions& opts,
+                               RandomAccessFileReader* reader, uint64_t offset,
+                               size_t length, size_t readahead_size,
+                               Env::IOPriority rate_limiter_priority,
+                               bool& copy_to_third_buffer);
+
   Status Read(const IOOptions& opts, RandomAccessFileReader* reader,
               Env::IOPriority rate_limiter_priority, uint64_t read_len,
               uint64_t chunk_len, uint64_t rounddown_start, uint32_t index);
@@ -256,6 +262,22 @@ class FilePrefetchBuffer {
     readahead_size_ = initial_auto_readahead_size_;
   }
 
+  bool IsEligibleForPrefetch(uint64_t offset, size_t n) {
+    // Prefetch only if this read is sequential otherwise reset readahead_size_
+    // to initial value.
+    if (!IsBlockSequential(offset)) {
+      UpdateReadPattern(offset, n, false /*decrease_readaheadsize*/);
+      ResetValues();
+      return false;
+    }
+    num_file_reads_++;
+    if (num_file_reads_ <= kMinNumFileReadsToStartAutoReadahead) {
+      UpdateReadPattern(offset, n, false /*decrease_readaheadsize*/);
+      return false;
+    }
+    return true;
+  }
+
   std::vector<BufferInfo> bufs_;
   // curr_ represents the index for bufs_ indicating which buffer is being
   // consumed currently.

package/deps/rocksdb/rocksdb/file/prefetch_test.cc

@@ -534,15 +534,24 @@ TEST_P(PrefetchTest, PrefetchWhenReseek) {
      * initially (2 more data blocks).
      */
     iter->Seek(BuildKey(0));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1000));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1004));  // Prefetch Data
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1008));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1011));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1015));  // Prefetch Data
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1019));
+    ASSERT_TRUE(iter->Valid());
     // Missed 2 blocks but they are already in buffer so no reset.
     iter->Seek(BuildKey(103));   // Already in buffer.
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1033));  // Prefetch Data
+    ASSERT_TRUE(iter->Valid());
     if (support_prefetch && !use_direct_io) {
       ASSERT_EQ(fs->GetPrefetchCount(), 3);
       fs->ClearPrefetchCount();
@@ -558,10 +567,15 @@ TEST_P(PrefetchTest, PrefetchWhenReseek) {
      */
     auto iter = std::unique_ptr<Iterator>(db_->NewIterator(ReadOptions()));
     iter->Seek(BuildKey(0));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1008));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1019));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1033));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1048));
+    ASSERT_TRUE(iter->Valid());
     if (support_prefetch && !use_direct_io) {
       ASSERT_EQ(fs->GetPrefetchCount(), 0);
       fs->ClearPrefetchCount();
@@ -576,9 +590,13 @@ TEST_P(PrefetchTest, PrefetchWhenReseek) {
      */
     auto iter = std::unique_ptr<Iterator>(db_->NewIterator(ReadOptions()));
     iter->Seek(BuildKey(0));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(10));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(100));
+    ASSERT_TRUE(iter->Valid());
     if (support_prefetch && !use_direct_io) {
       ASSERT_EQ(fs->GetPrefetchCount(), 0);
       fs->ClearPrefetchCount();
@@ -596,14 +614,21 @@ TEST_P(PrefetchTest, PrefetchWhenReseek) {
      */
     auto iter = std::unique_ptr<Iterator>(db_->NewIterator(ReadOptions()));
     iter->Seek(BuildKey(0));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1000));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1004));  // This iteration will prefetch buffer
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1008));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(
         BuildKey(996));  // Reseek won't prefetch any data and
                          // readahead_size will be initiallized to 8*1024.
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(992));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(989));
+    ASSERT_TRUE(iter->Valid());
     if (support_prefetch && !use_direct_io) {
       ASSERT_EQ(fs->GetPrefetchCount(), 1);
       fs->ClearPrefetchCount();
@@ -615,11 +640,17 @@ TEST_P(PrefetchTest, PrefetchWhenReseek) {
     // Read sequentially to confirm readahead_size is reset to initial value (2
     // more data blocks)
     iter->Seek(BuildKey(1011));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1015));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1019));  // Prefetch Data
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1022));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1026));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(103));  // Prefetch Data
+    ASSERT_TRUE(iter->Valid());
     if (support_prefetch && !use_direct_io) {
       ASSERT_EQ(fs->GetPrefetchCount(), 2);
       fs->ClearPrefetchCount();
@@ -634,12 +665,19 @@ TEST_P(PrefetchTest, PrefetchWhenReseek) {
      */
     auto iter = std::unique_ptr<Iterator>(db_->NewIterator(ReadOptions()));
     iter->Seek(BuildKey(0));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1167));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1334));  // This iteration will prefetch buffer
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1499));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1667));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1847));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1999));
+    ASSERT_TRUE(iter->Valid());
     if (support_prefetch && !use_direct_io) {
       ASSERT_EQ(fs->GetPrefetchCount(), 1);
       fs->ClearPrefetchCount();
@@ -766,8 +804,11 @@ TEST_P(PrefetchTest, PrefetchWhenReseekwithCache) {
     auto iter = std::unique_ptr<Iterator>(db_->NewIterator(ReadOptions()));
     // Warm up the cache
     iter->Seek(BuildKey(1011));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1015));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1019));
+    ASSERT_TRUE(iter->Valid());
     if (support_prefetch && !use_direct_io) {
       ASSERT_EQ(fs->GetPrefetchCount(), 1);
       fs->ClearPrefetchCount();
@@ -780,20 +821,31 @@ TEST_P(PrefetchTest, PrefetchWhenReseekwithCache) {
     // After caching, blocks will be read from cache (Sequential blocks)
     auto iter = std::unique_ptr<Iterator>(db_->NewIterator(ReadOptions()));
     iter->Seek(BuildKey(0));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1000));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1004));  // Prefetch data (not in cache).
+    ASSERT_TRUE(iter->Valid());
     // Missed one sequential block but next is in already in buffer so readahead
     // will not be reset.
     iter->Seek(BuildKey(1011));
+    ASSERT_TRUE(iter->Valid());
     // Prefetch data but blocks are in cache so no prefetch and reset.
     iter->Seek(BuildKey(1015));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1019));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1022));
+    ASSERT_TRUE(iter->Valid());
     // Prefetch data with readahead_size = 4 blocks.
     iter->Seek(BuildKey(1026));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(103));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1033));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1037));
+    ASSERT_TRUE(iter->Valid());
 
     if (support_prefetch && !use_direct_io) {
       ASSERT_EQ(fs->GetPrefetchCount(), 3);
@@ -881,7 +933,7 @@ TEST_P(PrefetchTest1, DBIterLevelReadAhead) {
         [&](void*) { buff_prefetch_count++; });
 
     SyncPoint::GetInstance()->SetCallBack(
-        "FilePrefetchBuffer::PrefetchAsync:Start",
+        "FilePrefetchBuffer::PrefetchAsyncInternal:Start",
         [&](void*) { buff_async_prefetch_count++; });
 
     // The callback checks, since reads are sequential, readahead_size doesn't
@@ -955,7 +1007,7 @@ class PrefetchTest2 : public DBTestBase,
 INSTANTIATE_TEST_CASE_P(PrefetchTest2, PrefetchTest2, ::testing::Bool());
 
 #ifndef ROCKSDB_LITE
-TEST_P(PrefetchTest2, NonSequentialReads) {
+TEST_P(PrefetchTest2, NonSequentialReadsWithAdaptiveReadahead) {
   const int kNumKeys = 1000;
   // Set options
   std::shared_ptr<MockFS> fs =
@@ -1002,9 +1054,8 @@ TEST_P(PrefetchTest2, NonSequentialReads) {
   int set_readahead = 0;
   size_t readahead_size = 0;
 
-  SyncPoint::GetInstance()->SetCallBack(
-      "FilePrefetchBuffer::PrefetchAsync:Start",
-      [&](void*) { buff_prefetch_count++; });
+  SyncPoint::GetInstance()->SetCallBack("FilePrefetchBuffer::Prefetch:Start",
+                                        [&](void*) { buff_prefetch_count++; });
   SyncPoint::GetInstance()->SetCallBack(
       "BlockPrefetcher::SetReadaheadState",
       [&](void* /*arg*/) { set_readahead++; });
@@ -1018,13 +1069,15 @@ TEST_P(PrefetchTest2, NonSequentialReads) {
     // Iterate until prefetch is done.
     ReadOptions ro;
     ro.adaptive_readahead = true;
-    // TODO akanksha: Remove after adding new units.
-    ro.async_io = true;
     auto iter = std::unique_ptr<Iterator>(db_->NewIterator(ro));
+
     iter->SeekToFirst();
+    ASSERT_TRUE(iter->Valid());
+
     while (iter->Valid() && buff_prefetch_count == 0) {
       iter->Next();
     }
+
     ASSERT_EQ(readahead_size, 8 * 1024);
     ASSERT_EQ(buff_prefetch_count, 1);
     ASSERT_EQ(set_readahead, 0);
@@ -1033,9 +1086,12 @@ TEST_P(PrefetchTest2, NonSequentialReads) {
     // Move to last file and check readahead size fallbacks to 8KB. So next
     // readahead size after prefetch should be 8 * 1024;
     iter->Seek(BuildKey(4004));
+    ASSERT_TRUE(iter->Valid());
+
     while (iter->Valid() && buff_prefetch_count == 0) {
       iter->Next();
     }
+
     ASSERT_EQ(readahead_size, 8 * 1024);
     ASSERT_EQ(set_readahead, 0);
     ASSERT_EQ(buff_prefetch_count, 1);
@@ -1099,7 +1155,7 @@ TEST_P(PrefetchTest2, DecreaseReadAheadIfInCache) {
   size_t decrease_readahead_size = 8 * 1024;
 
   SyncPoint::GetInstance()->SetCallBack(
-      "FilePrefetchBuffer::PrefetchAsync:Start",
+      "FilePrefetchBuffer::PrefetchAsyncInternal:Start",
       [&](void*) { buff_prefetch_count++; });
   SyncPoint::GetInstance()->SetCallBack(
       "FilePrefetchBuffer::TryReadFromCache", [&](void* arg) {
@@ -1120,8 +1176,11 @@ TEST_P(PrefetchTest2, DecreaseReadAheadIfInCache) {
     auto iter = std::unique_ptr<Iterator>(db_->NewIterator(ro));
     // Warm up the cache
     iter->Seek(BuildKey(1011));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1015));
+    ASSERT_TRUE(iter->Valid());
     iter->Seek(BuildKey(1019));
+    ASSERT_TRUE(iter->Valid());
     buff_prefetch_count = 0;
   }
 
@@ -1129,26 +1188,39 @@ TEST_P(PrefetchTest2, DecreaseReadAheadIfInCache) {
     ASSERT_OK(options.statistics->Reset());
     // After caching, blocks will be read from cache (Sequential blocks)
     auto iter = std::unique_ptr<Iterator>(db_->NewIterator(ro));
-    iter->Seek(BuildKey(0));
+    iter->Seek(
+        BuildKey(0));  // In cache so it will decrease the readahead_size.
     ASSERT_TRUE(iter->Valid());
-    iter->Seek(BuildKey(1000));
+    expected_current_readahead_size = std::max(
+        decrease_readahead_size,
+        (expected_current_readahead_size >= decrease_readahead_size
+             ? (expected_current_readahead_size - decrease_readahead_size)
+             : 0));
+
+    iter->Seek(BuildKey(1000));  // Prefetch the block.
     ASSERT_TRUE(iter->Valid());
-    iter->Seek(BuildKey(1004));  // Prefetch data (not in cache).
+    ASSERT_EQ(current_readahead_size, expected_current_readahead_size);
+    expected_current_readahead_size *= 2;
+
+    iter->Seek(BuildKey(1004));  // Prefetch the block.
     ASSERT_TRUE(iter->Valid());
     ASSERT_EQ(current_readahead_size, expected_current_readahead_size);
+    expected_current_readahead_size *= 2;
 
-    // Missed one sequential block but 1011 is already in buffer so
-    // readahead will not be reset.
+    // 1011 is already in cache but won't reset??
     iter->Seek(BuildKey(1011));
     ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ(current_readahead_size, expected_current_readahead_size);
 
     // Eligible to Prefetch data (not in buffer) but block is in cache so no
     // prefetch will happen and will result in decrease in readahead_size.
     // readahead_size will be 8 * 1024
     iter->Seek(BuildKey(1015));
     ASSERT_TRUE(iter->Valid());
-    expected_current_readahead_size -= decrease_readahead_size;
+    expected_current_readahead_size = std::max(
+        decrease_readahead_size,
+        (expected_current_readahead_size >= decrease_readahead_size
+             ? (expected_current_readahead_size - decrease_readahead_size)
+             : 0));
 
     // 1016 is the same block as 1015. So no change in readahead_size.
     iter->Seek(BuildKey(1016));
@@ -1169,7 +1241,7 @@ TEST_P(PrefetchTest2, DecreaseReadAheadIfInCache) {
     iter->Seek(BuildKey(1022));
     ASSERT_TRUE(iter->Valid());
     ASSERT_EQ(current_readahead_size, expected_current_readahead_size);
-    ASSERT_EQ(buff_prefetch_count, 2);
+    ASSERT_EQ(buff_prefetch_count, 3);
 
     // Check stats to make sure async prefetch is done.
     {
@@ -1179,6 +1251,7 @@ TEST_P(PrefetchTest2, DecreaseReadAheadIfInCache) {
         ASSERT_EQ(async_read_bytes.count, 0);
       } else {
         ASSERT_GT(async_read_bytes.count, 0);
+        ASSERT_GT(get_perf_context()->number_async_seek, 0);
       }
     }
 
@@ -1264,7 +1337,7 @@ TEST_P(PrefetchTestWithPosix, ReadAsyncWithPosixFS) {
   }
 
   SyncPoint::GetInstance()->SetCallBack(
-      "FilePrefetchBuffer::PrefetchAsync:Start",
+      "FilePrefetchBuffer::PrefetchAsyncInternal:Start",
       [&](void*) { buff_prefetch_count++; });
 
   SyncPoint::GetInstance()->SetCallBack(
@@ -1275,12 +1348,15 @@ TEST_P(PrefetchTestWithPosix, ReadAsyncWithPosixFS) {
   // Read the keys.
   {
     ASSERT_OK(options.statistics->Reset());
+    get_perf_context()->Reset();
+
     auto iter = std::unique_ptr<Iterator>(db_->NewIterator(ro));
     int num_keys = 0;
     for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
       ASSERT_OK(iter->status());
       num_keys++;
     }
+
     ASSERT_EQ(num_keys, total_keys);
     ASSERT_GT(buff_prefetch_count, 0);
 
@@ -1301,6 +1377,55 @@ TEST_P(PrefetchTestWithPosix, ReadAsyncWithPosixFS) {
       }
       ASSERT_GT(prefetched_bytes_discarded.count, 0);
     }
+    ASSERT_EQ(get_perf_context()->number_async_seek, 0);
+  }
+
+  {
+    // Read the keys using seek.
+    {
+      ASSERT_OK(options.statistics->Reset());
+      get_perf_context()->Reset();
+
+      auto iter = std::unique_ptr<Iterator>(db_->NewIterator(ro));
+      int num_keys = 0;
+      iter->Seek(BuildKey(450));
+      while (iter->Valid()) {
+        ASSERT_OK(iter->status());
+        num_keys++;
+        iter->Next();
+      }
+      ASSERT_OK(iter->status());
+
+      iter->Seek(BuildKey(450));
+      while (iter->Valid()) {
+        ASSERT_OK(iter->status());
+        num_keys++;
+        iter->Prev();
+      }
+
+      ASSERT_EQ(num_keys, total_keys + 1);
+      ASSERT_GT(buff_prefetch_count, 0);
+
+      // Check stats to make sure async prefetch is done.
+      {
+        HistogramData async_read_bytes;
+        options.statistics->histogramData(ASYNC_READ_BYTES, &async_read_bytes);
+        HistogramData prefetched_bytes_discarded;
+        options.statistics->histogramData(PREFETCHED_BYTES_DISCARDED,
+                                          &prefetched_bytes_discarded);
+
+        // Not all platforms support iouring. In that case, ReadAsync in posix
+        // won't submit async requests.
+        if (read_async_called) {
+          ASSERT_GT(async_read_bytes.count, 0);
+          ASSERT_GT(get_perf_context()->number_async_seek, 0);
+        } else {
+          ASSERT_EQ(async_read_bytes.count, 0);
+          ASSERT_EQ(get_perf_context()->number_async_seek, 0);
+        }
+        ASSERT_GT(prefetched_bytes_discarded.count, 0);
+      }
+    }
   }
 
   SyncPoint::GetInstance()->DisableProcessing();

package/deps/rocksdb/rocksdb/file/writable_file_writer.cc

@@ -55,9 +55,9 @@ IOStatus WritableFileWriter::Append(const Slice& data, uint32_t crc32c_checksum,
 
   {
     IOOptions io_options;
-    WritableFileWriter::DecideRateLimiterPriority(
-        writable_file_->GetIOPriority(), op_rate_limiter_priority,
-        io_options.rate_limiter_priority);
+    io_options.rate_limiter_priority =
+        WritableFileWriter::DecideRateLimiterPriority(
+            writable_file_->GetIOPriority(), op_rate_limiter_priority);
     IOSTATS_TIMER_GUARD(prepare_write_nanos);
     TEST_SYNC_POINT("WritableFileWriter::Append:BeforePrepareWrite");
     writable_file_->PrepareWrite(static_cast<size_t>(GetFileSize()), left,
@@ -338,9 +338,9 @@ IOStatus WritableFileWriter::Flush(Env::IOPriority op_rate_limiter_priority) {
     }
 #endif
     IOOptions io_options;
-    WritableFileWriter::DecideRateLimiterPriority(
-        writable_file_->GetIOPriority(), op_rate_limiter_priority,
-        io_options.rate_limiter_priority);
+    io_options.rate_limiter_priority =
+        WritableFileWriter::DecideRateLimiterPriority(
+            writable_file_->GetIOPriority(), op_rate_limiter_priority);
     s = writable_file_->Flush(io_options, nullptr);
 #ifndef ROCKSDB_LITE
     if (ShouldNotifyListeners()) {
@@ -507,11 +507,11 @@ IOStatus WritableFileWriter::WriteBuffered(
   size_t left = size;
   DataVerificationInfo v_info;
   char checksum_buf[sizeof(uint32_t)];
-  IOOptions io_options;
   Env::IOPriority rate_limiter_priority_used =
       WritableFileWriter::DecideRateLimiterPriority(
-          writable_file_->GetIOPriority(), op_rate_limiter_priority,
-          io_options.rate_limiter_priority);
+          writable_file_->GetIOPriority(), op_rate_limiter_priority);
+  IOOptions io_options;
+  io_options.rate_limiter_priority = rate_limiter_priority_used;
 
   while (left > 0) {
     size_t allowed = left;
@@ -596,11 +596,11 @@ IOStatus WritableFileWriter::WriteBufferedWithChecksum(
   size_t left = size;
   DataVerificationInfo v_info;
   char checksum_buf[sizeof(uint32_t)];
-  IOOptions io_options;
   Env::IOPriority rate_limiter_priority_used =
       WritableFileWriter::DecideRateLimiterPriority(
-          writable_file_->GetIOPriority(), op_rate_limiter_priority,
-          io_options.rate_limiter_priority);
+          writable_file_->GetIOPriority(), op_rate_limiter_priority);
+  IOOptions io_options;
+  io_options.rate_limiter_priority = rate_limiter_priority_used;
   // Check how much is allowed. Here, we loop until the rate limiter allows to
   // write the entire buffer.
   // TODO: need to be improved since it sort of defeats the purpose of the rate
@@ -726,11 +726,11 @@ IOStatus WritableFileWriter::WriteDirect(
   size_t left = buf_.CurrentSize();
   DataVerificationInfo v_info;
   char checksum_buf[sizeof(uint32_t)];
-  IOOptions io_options;
   Env::IOPriority rate_limiter_priority_used =
       WritableFileWriter::DecideRateLimiterPriority(
-          writable_file_->GetIOPriority(), op_rate_limiter_priority,
-          io_options.rate_limiter_priority);
+          writable_file_->GetIOPriority(), op_rate_limiter_priority);
+  IOOptions io_options;
+  io_options.rate_limiter_priority = rate_limiter_priority_used;
 
   while (left > 0) {
     // Check how much is allowed
@@ -827,11 +827,11 @@ IOStatus WritableFileWriter::WriteDirectWithChecksum(
   DataVerificationInfo v_info;
   char checksum_buf[sizeof(uint32_t)];
 
-  IOOptions io_options;
   Env::IOPriority rate_limiter_priority_used =
       WritableFileWriter::DecideRateLimiterPriority(
-          writable_file_->GetIOPriority(), op_rate_limiter_priority,
-          io_options.rate_limiter_priority);
+          writable_file_->GetIOPriority(), op_rate_limiter_priority);
+  IOOptions io_options;
+  io_options.rate_limiter_priority = rate_limiter_priority_used;
   // Check how much is allowed. Here, we loop until the rate limiter allows to
   // write the entire buffer.
   // TODO: need to be improved since it sort of defeats the purpose of the rate
@@ -901,21 +901,17 @@ IOStatus WritableFileWriter::WriteDirectWithChecksum(
 #endif  // !ROCKSDB_LITE
 Env::IOPriority WritableFileWriter::DecideRateLimiterPriority(
     Env::IOPriority writable_file_io_priority,
-    Env::IOPriority op_rate_limiter_priority,
-    Env::IOPriority& iooptions_io_priority) {
-  Env::IOPriority rate_limiter_priority{Env::IO_TOTAL};
+    Env::IOPriority op_rate_limiter_priority) {
   if (writable_file_io_priority == Env::IO_TOTAL &&
       op_rate_limiter_priority == Env::IO_TOTAL) {
-    rate_limiter_priority = Env::IO_TOTAL;
+    return Env::IO_TOTAL;
   } else if (writable_file_io_priority == Env::IO_TOTAL) {
-    rate_limiter_priority = op_rate_limiter_priority;
+    return op_rate_limiter_priority;
   } else if (op_rate_limiter_priority == Env::IO_TOTAL) {
-    rate_limiter_priority = writable_file_io_priority;
+    return writable_file_io_priority;
   } else {
-    rate_limiter_priority = op_rate_limiter_priority;
+    return op_rate_limiter_priority;
   }
-  iooptions_io_priority = rate_limiter_priority;
-  return rate_limiter_priority;
 }
 
 }  // namespace ROCKSDB_NAMESPACE

package/deps/rocksdb/rocksdb/file/writable_file_writer.h

@@ -277,11 +277,10 @@ class WritableFileWriter {
   const char* GetFileChecksumFuncName() const;
 
  private:
-  // Decide the Rate Limiter priority and update io_options.io_priority.
+  // Decide the Rate Limiter priority.
   static Env::IOPriority DecideRateLimiterPriority(
       Env::IOPriority writable_file_io_priority,
-      Env::IOPriority op_rate_limiter_priority,
-      Env::IOPriority& iooptions_io_priority);
+      Env::IOPriority op_rate_limiter_priority);
 
   // Used when os buffering is OFF and we are writing
   // DMA such as in Direct I/O mode

package/deps/rocksdb/rocksdb/include/rocksdb/advanced_options.h

@@ -100,8 +100,9 @@ struct CompressionOptions {
   //
   // The dictionary is created by sampling the SST file data. If
   // `zstd_max_train_bytes` is nonzero, the samples are passed through zstd's
-  // dictionary generator. Otherwise, the random samples are used directly as
-  // the dictionary.
+  // dictionary generator (see comments for option `use_zstd_dict_trainer` for
+  // detail on dictionary generator). If `zstd_max_train_bytes` is zero, the
+  // random samples are used directly as the dictionary.
   //
   // When compression dictionary is disabled, we compress and write each block
   // before buffering data for the next one. When compression dictionary is
@@ -173,6 +174,20 @@ struct CompressionOptions {
   // Default: 0 (unlimited)
   uint64_t max_dict_buffer_bytes;
 
+  // Use zstd trainer to generate dictionaries. When this option is set to true,
+  // zstd_max_train_bytes of training data sampled from max_dict_buffer_bytes
+  // buffered data will be passed to zstd dictionary trainer to generate a
+  // dictionary of size max_dict_bytes.
+  //
+  // When this option is false, zstd's API ZDICT_finalizeDictionary() will be
+  // called to generate dictionaries. zstd_max_train_bytes of training sampled
+  // data will be passed to this API. Using this API should save CPU time on
+  // dictionary training, but the compression ratio may not be as good as using
+  // a dictionary trainer.
+  //
+  // Default: true
+  bool use_zstd_dict_trainer;
+
   CompressionOptions()
       : window_bits(-14),
         level(kDefaultCompressionLevel),
@@ -181,11 +196,13 @@ struct CompressionOptions {
         zstd_max_train_bytes(0),
         parallel_threads(1),
         enabled(false),
-        max_dict_buffer_bytes(0) {}
+        max_dict_buffer_bytes(0),
+        use_zstd_dict_trainer(true) {}
   CompressionOptions(int wbits, int _lev, int _strategy,
                      uint32_t _max_dict_bytes, uint32_t _zstd_max_train_bytes,
                      uint32_t _parallel_threads, bool _enabled,
-                     uint64_t _max_dict_buffer_bytes)
+                     uint64_t _max_dict_buffer_bytes,
+                     bool _use_zstd_dict_trainer)
       : window_bits(wbits),
         level(_lev),
         strategy(_strategy),
@@ -193,7 +210,8 @@ struct CompressionOptions {
         zstd_max_train_bytes(_zstd_max_train_bytes),
         parallel_threads(_parallel_threads),
         enabled(_enabled),
-        max_dict_buffer_bytes(_max_dict_buffer_bytes) {}
+        max_dict_buffer_bytes(_max_dict_buffer_bytes),
+        use_zstd_dict_trainer(_use_zstd_dict_trainer) {}
 };
 
 // Temperature of a file. Used to pass to FileSystem for a different