@nxtedition/rocksdb 7.1.14 → 7.1.16

package/deps/rocksdb/rocksdb/file/file_prefetch_buffer.h

@@ -20,24 +20,42 @@
 #include "rocksdb/file_system.h"
 #include "rocksdb/options.h"
 #include "util/aligned_buffer.h"
+#include "util/autovector.h"
+#include "util/stop_watch.h"
 
 namespace ROCKSDB_NAMESPACE {
 
-#define DEAFULT_DECREMENT 8 * 1024
+#define DEFAULT_DECREMENT 8 * 1024
 
 struct IOOptions;
 class RandomAccessFileReader;
 
 struct BufferInfo {
   AlignedBuffer buffer_;
+
   uint64_t offset_ = 0;
+
+  // Below parameters are used in case of async read flow.
+  // Length requested for in ReadAsync.
+  size_t async_req_len_ = 0;
+
+  // async_read_in_progress can be used as mutex. Callback can update the buffer
+  // and its size but async_read_in_progress is only set by main thread.
+  bool async_read_in_progress_ = false;
+
+  // io_handle is allocated and used by underlying file system in case of
+  // asynchronous reads.
+  void* io_handle_ = nullptr;
+
+  IOHandleDeleter del_fn_ = nullptr;
+
+  // pos represents the index of this buffer in vector of BufferInfo.
+  uint32_t pos_ = 0;
 };
 
 // FilePrefetchBuffer is a smart buffer to store and read data from a file.
 class FilePrefetchBuffer {
  public:
-  static const int kMinNumFileReadsToStartAutoReadahead = 2;
-
   // Constructor.
   //
   // All arguments are optional.
@@ -54,9 +72,6 @@ class FilePrefetchBuffer {
   //   it. Used for adaptable readahead of the file footer/metadata.
   // implicit_auto_readahead : Readahead is enabled implicitly by rocksdb after
   //   doing sequential scans for two times.
-  // async_io : When async_io is enabled, if it's implicit_auto_readahead, it
-  //   prefetches data asynchronously in second buffer while curr_ is being
-  //   consumed.
   //
   // Automatic readhead is enabled for a file if readahead_size
   // and max_readahead_size are passed in.
@@ -65,8 +80,10 @@ class FilePrefetchBuffer {
   FilePrefetchBuffer(size_t readahead_size = 0, size_t max_readahead_size = 0,
                      bool enable = true, bool track_min_offset = false,
                      bool implicit_auto_readahead = false,
-                     uint64_t num_file_reads = 0, FileSystem* fs = nullptr,
-                     SystemClock* clock = nullptr, Statistics* stats = nullptr)
+                     uint64_t num_file_reads = 0,
+                     uint64_t num_file_reads_for_auto_readahead = 0,
+                     FileSystem* fs = nullptr, SystemClock* clock = nullptr,
+                     Statistics* stats = nullptr)
       : curr_(0),
         readahead_size_(readahead_size),
         initial_auto_readahead_size_(readahead_size),
@@ -77,47 +94,83 @@ class FilePrefetchBuffer {
         implicit_auto_readahead_(implicit_auto_readahead),
         prev_offset_(0),
         prev_len_(0),
+        num_file_reads_for_auto_readahead_(num_file_reads_for_auto_readahead),
         num_file_reads_(num_file_reads),
-        io_handle_(nullptr),
-        del_fn_(nullptr),
-        async_read_in_progress_(false),
-        async_request_submitted_(false),
+        explicit_prefetch_submitted_(false),
         fs_(fs),
         clock_(clock),
         stats_(stats) {
-    assert((num_file_reads_ >= kMinNumFileReadsToStartAutoReadahead + 1) ||
+    assert((num_file_reads_ >= num_file_reads_for_auto_readahead_ + 1) ||
            (num_file_reads_ == 0));
-    // If async_io_ is enabled, data is asynchronously filled in second buffer
-    // while curr_ is being consumed. If data is overlapping in two buffers,
-    // data is copied to third buffer to return continuous buffer.
+    // If ReadOptions.async_io is enabled, data is asynchronously filled in
+    // second buffer while curr_ is being consumed. If data is overlapping in
+    // two buffers, data is copied to third buffer to return continuous buffer.
     bufs_.resize(3);
+    for (uint32_t i = 0; i < 2; i++) {
+      bufs_[i].pos_ = i;
+    }
   }
 
   ~FilePrefetchBuffer() {
     // Abort any pending async read request before destroying the class object.
-    if (async_read_in_progress_ && fs_ != nullptr) {
+    if (fs_ != nullptr) {
       std::vector<void*> handles;
-      handles.emplace_back(io_handle_);
-      Status s = fs_->AbortIO(handles);
-      assert(s.ok());
+      for (uint32_t i = 0; i < 2; i++) {
+        if (bufs_[i].async_read_in_progress_ &&
+            bufs_[i].io_handle_ != nullptr) {
+          handles.emplace_back(bufs_[i].io_handle_);
+        }
+      }
+      if (!handles.empty()) {
+        StopWatch sw(clock_, stats_, ASYNC_PREFETCH_ABORT_MICROS);
+        Status s = fs_->AbortIO(handles);
+        assert(s.ok());
+      }
     }
 
     // Prefetch buffer bytes discarded.
     uint64_t bytes_discarded = 0;
-    if (bufs_[curr_].buffer_.CurrentSize() != 0) {
-      bytes_discarded = bufs_[curr_].buffer_.CurrentSize();
-    }
-    if (bufs_[curr_ ^ 1].buffer_.CurrentSize() != 0) {
-      bytes_discarded += bufs_[curr_ ^ 1].buffer_.CurrentSize();
+    // Iterated over 2 buffers.
+    for (int i = 0; i < 2; i++) {
+      int first = i;
+      int second = i ^ 1;
+
+      if (DoesBufferContainData(first)) {
+        // If last block was read completely from first and some bytes in
+        // first buffer are still unconsumed.
+        if (prev_offset_ >= bufs_[first].offset_ &&
+            prev_offset_ + prev_len_ <
+                bufs_[first].offset_ + bufs_[first].buffer_.CurrentSize()) {
+          bytes_discarded += bufs_[first].buffer_.CurrentSize() -
+                             (prev_offset_ + prev_len_ - bufs_[first].offset_);
+        }
+        // If data was in second buffer and some/whole block bytes were read
+        // from second buffer.
+        else if (prev_offset_ < bufs_[first].offset_ &&
+                 !DoesBufferContainData(second)) {
+          // If last block read was completely from different buffer, this
+          // buffer is unconsumed.
+          if (prev_offset_ + prev_len_ <= bufs_[first].offset_) {
+            bytes_discarded += bufs_[first].buffer_.CurrentSize();
+          }
+          // If last block read overlaps with this buffer and some data is
+          // still unconsumed and previous buffer (second) is not cleared.
+          else if (prev_offset_ + prev_len_ > bufs_[first].offset_ &&
+                   bufs_[first].offset_ + bufs_[first].buffer_.CurrentSize() ==
+                       bufs_[second].offset_) {
+            bytes_discarded += bufs_[first].buffer_.CurrentSize() -
+                               (/*bytes read from this buffer=*/prev_len_ -
+                                (bufs_[first].offset_ - prev_offset_));
+          }
+        }
+      }
     }
-    RecordInHistogram(stats_, PREFETCHED_BYTES_DISCARDED, bytes_discarded);
 
-    // Release io_handle_.
-    if (io_handle_ != nullptr && del_fn_ != nullptr) {
-      del_fn_(io_handle_);
-      io_handle_ = nullptr;
-      del_fn_ = nullptr;
+    for (uint32_t i = 0; i < 2; i++) {
+      // Release io_handle.
+      DestroyAndClearIOHandle(i);
     }
+    RecordInHistogram(stats_, PREFETCHED_BYTES_DISCARDED, bytes_discarded);
   }
 
   // Load data into the buffer from a file.
@@ -126,9 +179,6 @@ class FilePrefetchBuffer {
   // n                     : the number of bytes to read.
   // rate_limiter_priority : rate limiting priority, or `Env::IO_TOTAL` to
   //                         bypass.
-  // is_async_read         : if the data should be prefetched by calling read
-  //                         asynchronously. It should be set true when called
-  //                         from TryReadFromCache.
   Status Prefetch(const IOOptions& opts, RandomAccessFileReader* reader,
                   uint64_t offset, size_t n,
                   Env::IOPriority rate_limiter_priority);
@@ -194,7 +244,7 @@ class FilePrefetchBuffer {
   }
 
   void DecreaseReadAheadIfEligible(uint64_t offset, size_t size,
-                                   size_t value = DEAFULT_DECREMENT) {
+                                   size_t value = DEFAULT_DECREMENT) {
     // Decrease the readahead_size if
     // - its enabled internally by RocksDB (implicit_auto_readahead_) and,
     // - readahead_size is greater than 0 and,
@@ -203,12 +253,14 @@ class FilePrefetchBuffer {
     //   - few/no bytes are in buffer and,
     //   - block is sequential with the previous read and,
     //   - num_file_reads_ + 1 (including this read) >
-    //   kMinNumFileReadsToStartAutoReadahead
+    //   num_file_reads_for_auto_readahead_
+    size_t curr_size = bufs_[curr_].async_read_in_progress_
+                           ? bufs_[curr_].async_req_len_
+                           : bufs_[curr_].buffer_.CurrentSize();
     if (implicit_auto_readahead_ && readahead_size_ > 0) {
-      if ((offset + size >
-           bufs_[curr_].offset_ + bufs_[curr_].buffer_.CurrentSize()) &&
+      if ((offset + size > bufs_[curr_].offset_ + curr_size) &&
           IsBlockSequential(offset) &&
-          (num_file_reads_ + 1 > kMinNumFileReadsToStartAutoReadahead)) {
+          (num_file_reads_ + 1 > num_file_reads_for_auto_readahead_)) {
         readahead_size_ =
             std::max(initial_auto_readahead_size_,
                      (readahead_size_ >= value ? readahead_size_ - value : 0));
@@ -224,8 +276,14 @@ class FilePrefetchBuffer {
   // and data present in buffer_. It also allocates new buffer or refit tail if
   // required.
   void CalculateOffsetAndLen(size_t alignment, uint64_t offset,
-                             size_t roundup_len, size_t index, bool refit_tail,
-                             uint64_t& chunk_len);
+                             size_t roundup_len, uint32_t index,
+                             bool refit_tail, uint64_t& chunk_len);
+
+  void AbortIOIfNeeded(uint64_t offset);
+
+  void AbortAllIOs();
+
+  void UpdateBuffersIfNeeded(uint64_t offset);
 
   // It calls Poll API if any there is any pending asynchronous request. It then
   // checks if data is in any buffer. It clears the outdated data and swaps the
@@ -243,8 +301,7 @@ class FilePrefetchBuffer {
               uint64_t chunk_len, uint64_t rounddown_start, uint32_t index);
 
   Status ReadAsync(const IOOptions& opts, RandomAccessFileReader* reader,
-                   uint64_t read_len, uint64_t chunk_len,
-                   uint64_t rounddown_start, uint32_t index);
+                   uint64_t read_len, uint64_t rounddown_start, uint32_t index);
 
   // Copy the data from src to third buffer.
   void CopyDataToBuffer(uint32_t src, uint64_t& offset, size_t& length);
@@ -273,25 +330,60 @@ class FilePrefetchBuffer {
     // Since async request was submitted in last call directly by calling
     // PrefetchAsync, it skips num_file_reads_ check as this call is to poll the
     // data submitted in previous call.
-    if (async_request_submitted_) {
+    if (explicit_prefetch_submitted_) {
       return true;
     }
-    if (num_file_reads_ <= kMinNumFileReadsToStartAutoReadahead) {
+    if (num_file_reads_ <= num_file_reads_for_auto_readahead_) {
       UpdateReadPattern(offset, n, false /*decrease_readaheadsize*/);
       return false;
     }
     return true;
   }
 
+  // Helper functions.
+  bool IsDataBlockInBuffer(uint64_t offset, size_t length, uint32_t index) {
+    return (offset >= bufs_[index].offset_ &&
+            offset + length <=
+                bufs_[index].offset_ + bufs_[index].buffer_.CurrentSize());
+  }
+  bool IsOffsetInBuffer(uint64_t offset, uint32_t index) {
+    return (offset >= bufs_[index].offset_ &&
+            offset < bufs_[index].offset_ + bufs_[index].buffer_.CurrentSize());
+  }
+  bool DoesBufferContainData(uint32_t index) {
+    return bufs_[index].buffer_.CurrentSize() > 0;
+  }
+  bool IsBufferOutdated(uint64_t offset, uint32_t index) {
+    return (
+        !bufs_[index].async_read_in_progress_ && DoesBufferContainData(index) &&
+        offset >= bufs_[index].offset_ + bufs_[index].buffer_.CurrentSize());
+  }
+  bool IsBufferOutdatedWithAsyncProgress(uint64_t offset, uint32_t index) {
+    return (bufs_[index].async_read_in_progress_ &&
+            bufs_[index].io_handle_ != nullptr &&
+            offset >= bufs_[index].offset_ + bufs_[index].async_req_len_);
+  }
+
+  void DestroyAndClearIOHandle(uint32_t index) {
+    if (bufs_[index].io_handle_ != nullptr && bufs_[index].del_fn_ != nullptr) {
+      bufs_[index].del_fn_(bufs_[index].io_handle_);
+      bufs_[index].io_handle_ = nullptr;
+      bufs_[index].del_fn_ = nullptr;
+    }
+    bufs_[index].async_read_in_progress_ = false;
+  }
+
   std::vector<BufferInfo> bufs_;
   // curr_ represents the index for bufs_ indicating which buffer is being
   // consumed currently.
   uint32_t curr_;
+
   size_t readahead_size_;
   size_t initial_auto_readahead_size_;
   // FilePrefetchBuffer object won't be created from Iterator flow if
   // max_readahead_size_ = 0.
   size_t max_readahead_size_;
+
   // The minimum `offset` ever passed to TryReadFromCache().
   size_t min_offset_read_;
   // if false, TryReadFromCache() always return false, and we only take stats
@@ -306,20 +398,16 @@ class FilePrefetchBuffer {
   bool implicit_auto_readahead_;
   uint64_t prev_offset_;
   size_t prev_len_;
-  // num_file_reads_ is only used when implicit_auto_readahead_ is set.
+  // num_file_reads_ and num_file_reads_for_auto_readahead_ is only used when
+  // implicit_auto_readahead_ is set.
+  uint64_t num_file_reads_for_auto_readahead_;
   uint64_t num_file_reads_;
 
-  // io_handle_ is allocated and used by underlying file system in case of
-  // asynchronous reads.
-  void* io_handle_;
-  IOHandleDeleter del_fn_;
-  bool async_read_in_progress_;
-
-  // If async_request_submitted_ is set then it indicates RocksDB called
-  // PrefetchAsync to submit request. It needs to TryReadFromCacheAsync to poll
-  // the submitted request without checking if data is sequential and
+  // If explicit_prefetch_submitted_ is set then it indicates RocksDB called
+  // PrefetchAsync to submit request. It needs to call TryReadFromCacheAsync to
+  // poll the submitted request without checking if data is sequential and
   // num_file_reads_.
-  bool async_request_submitted_;
+  bool explicit_prefetch_submitted_;
 
   FileSystem* fs_;
   SystemClock* clock_;

package/deps/rocksdb/rocksdb/file/filename.cc

@@ -388,7 +388,7 @@ bool ParseFileName(const std::string& fname, uint64_t* number,
 
 IOStatus SetCurrentFile(FileSystem* fs, const std::string& dbname,
                         uint64_t descriptor_number,
-                        FSDirectory* directory_to_fsync) {
+                        FSDirectory* dir_contains_current_file) {
   // Remove leading "dbname/" and add newline to manifest file name
   std::string manifest = DescriptorFileName(dbname, descriptor_number);
   Slice contents = manifest;
@@ -404,8 +404,8 @@ IOStatus SetCurrentFile(FileSystem* fs, const std::string& dbname,
     TEST_SYNC_POINT_CALLBACK("SetCurrentFile:AfterRename", &s);
   }
   if (s.ok()) {
-    if (directory_to_fsync != nullptr) {
-      s = directory_to_fsync->FsyncWithDirOptions(
+    if (dir_contains_current_file != nullptr) {
+      s = dir_contains_current_file->FsyncWithDirOptions(
           IOOptions(), nullptr, DirFsyncOptions(CurrentFileName(dbname)));
     }
   } else {

package/deps/rocksdb/rocksdb/file/filename.h

@@ -160,10 +160,12 @@ extern bool ParseFileName(const std::string& filename, uint64_t* number,
                           FileType* type, WalFileType* log_type = nullptr);
 
 // Make the CURRENT file point to the descriptor file with the
-// specified number.
+// specified number. On its success and when dir_contains_current_file is not
+// nullptr, the function will fsync the directory containing the CURRENT file
+// when
 extern IOStatus SetCurrentFile(FileSystem* fs, const std::string& dbname,
                                uint64_t descriptor_number,
-                               FSDirectory* directory_to_fsync);
+                               FSDirectory* dir_contains_current_file);
 
 // Make the IDENTITY file for the db
 extern Status SetIdentityFile(Env* env, const std::string& dbname,