@nxtedition/rocksdb 8.2.8 → 9.0.1

package/deps/rocksdb/rocksdb/file/random_access_file_reader.cc

@@ -22,12 +22,37 @@
 #include "util/rate_limiter_impl.h"
 
 namespace ROCKSDB_NAMESPACE {
-const std::array<Histograms, std::size_t(Env::IOActivity::kUnknown)>
-    kReadHistograms{{
-        FILE_READ_FLUSH_MICROS,
-        FILE_READ_COMPACTION_MICROS,
-        FILE_READ_DB_OPEN_MICROS,
-    }};
+inline Histograms GetFileReadHistograms(Statistics* stats,
+                                        Env::IOActivity io_activity) {
+  switch (io_activity) {
+    case Env::IOActivity::kFlush:
+      return Histograms::FILE_READ_FLUSH_MICROS;
+    case Env::IOActivity::kCompaction:
+      return Histograms::FILE_READ_COMPACTION_MICROS;
+    case Env::IOActivity::kDBOpen:
+      return Histograms::FILE_READ_DB_OPEN_MICROS;
+    default:
+      break;
+  }
+
+  if (stats && stats->get_stats_level() > StatsLevel::kExceptDetailedTimers) {
+    switch (io_activity) {
+      case Env::IOActivity::kGet:
+        return Histograms::FILE_READ_GET_MICROS;
+      case Env::IOActivity::kMultiGet:
+        return Histograms::FILE_READ_MULTIGET_MICROS;
+      case Env::IOActivity::kDBIterator:
+        return Histograms::FILE_READ_DB_ITERATOR_MICROS;
+      case Env::IOActivity::kVerifyDBChecksum:
+        return Histograms::FILE_READ_VERIFY_DB_CHECKSUM_MICROS;
+      case Env::IOActivity::kVerifyFileChecksums:
+        return Histograms::FILE_READ_VERIFY_FILE_CHECKSUMS_MICROS;
+      default:
+        break;
+    }
+  }
+  return Histograms::HISTOGRAM_ENUM_MAX;
+}
 inline void RecordIOStats(Statistics* stats, Temperature file_temperature,
                           bool is_last_level, size_t size) {
   IOSTATS_ADD(bytes_read, size);
@@ -79,11 +104,11 @@ IOStatus RandomAccessFileReader::Create(
   return io_s;
 }
 
-IOStatus RandomAccessFileReader::Read(
-    const IOOptions& opts, uint64_t offset, size_t n, Slice* result,
-    char* scratch, AlignedBuf* aligned_buf,
-    Env::IOPriority rate_limiter_priority) const {
+IOStatus RandomAccessFileReader::Read(const IOOptions& opts, uint64_t offset,
+                                      size_t n, Slice* result, char* scratch,
+                                      AlignedBuf* aligned_buf) const {
   (void)aligned_buf;
+  const Env::IOPriority rate_limiter_priority = opts.rate_limiter_priority;
 
   TEST_SYNC_POINT_CALLBACK("RandomAccessFileReader::Read", nullptr);
 
@@ -97,17 +122,23 @@ IOStatus RandomAccessFileReader::Read(
 
   IOStatus io_s;
   uint64_t elapsed = 0;
+  size_t alignment = file_->GetRequiredBufferAlignment();
+  bool is_aligned = false;
+  if (scratch != nullptr) {
+    // Check if offset, length and buffer are aligned.
+    is_aligned = (offset & (alignment - 1)) == 0 &&
+                 (n & (alignment - 1)) == 0 &&
+                 (uintptr_t(scratch) & (alignment - 1)) == 0;
+  }
+
   {
     StopWatch sw(clock_, stats_, hist_type_,
-                 (opts.io_activity != Env::IOActivity::kUnknown)
-                     ? kReadHistograms[(std::size_t)(opts.io_activity)]
-                     : Histograms::HISTOGRAM_ENUM_MAX,
+                 GetFileReadHistograms(stats_, opts.io_activity),
                  (stats_ != nullptr) ? &elapsed : nullptr, true /*overwrite*/,
                  true /*delay_enabled*/);
     auto prev_perf_level = GetPerfLevel();
     IOSTATS_TIMER_GUARD(read_nanos);
-    if (use_direct_io()) {
-      size_t alignment = file_->GetRequiredBufferAlignment();
+    if (use_direct_io() && is_aligned == false) {
       size_t aligned_offset =
           TruncateToPageBoundary(alignment, static_cast<size_t>(offset));
       size_t offset_advance = static_cast<size_t>(offset) - aligned_offset;
@@ -182,9 +213,9 @@ IOStatus RandomAccessFileReader::Read(
           if (rate_limiter_->IsRateLimited(RateLimiter::OpType::kRead)) {
             sw.DelayStart();
           }
-          allowed = rate_limiter_->RequestToken(n - pos, 0 /* alignment */,
-                                                rate_limiter_priority, stats_,
-                                                RateLimiter::OpType::kRead);
+          allowed = rate_limiter_->RequestToken(
+              n - pos, (use_direct_io() ? alignment : 0), rate_limiter_priority,
+              stats_, RateLimiter::OpType::kRead);
           if (rate_limiter_->IsRateLimited(RateLimiter::OpType::kRead)) {
             sw.DelayStop();
           }
@@ -240,6 +271,14 @@ IOStatus RandomAccessFileReader::Read(
     file_read_hist_->Add(elapsed);
   }
 
+#ifndef NDEBUG
+  auto pair = std::make_pair(&file_name_, &io_s);
+  if (offset == 0) {
+    TEST_SYNC_POINT_CALLBACK("RandomAccessFileReader::Read::BeforeReturn",
+                             &pair);
+  }
+  TEST_SYNC_POINT_CALLBACK("RandomAccessFileReader::Read::AnyOffset", &pair);
+#endif
   return io_s;
 }
 
@@ -269,9 +308,10 @@ bool TryMerge(FSReadRequest* dest, const FSReadRequest& src) {
   return true;
 }
 
-IOStatus RandomAccessFileReader::MultiRead(
-    const IOOptions& opts, FSReadRequest* read_reqs, size_t num_reqs,
-    AlignedBuf* aligned_buf, Env::IOPriority rate_limiter_priority) const {
+IOStatus RandomAccessFileReader::MultiRead(const IOOptions& opts,
+                                           FSReadRequest* read_reqs,
+                                           size_t num_reqs,
+                                           AlignedBuf* aligned_buf) const {
   (void)aligned_buf;  // suppress warning of unused variable in LITE mode
   assert(num_reqs > 0);
 
@@ -280,6 +320,7 @@ IOStatus RandomAccessFileReader::MultiRead(
     assert(read_reqs[i].offset <= read_reqs[i + 1].offset);
   }
 #endif  // !NDEBUG
+  const Env::IOPriority rate_limiter_priority = opts.rate_limiter_priority;
 
   // To be paranoid modify scratch a little bit, so in case underlying
   // FileSystem doesn't fill the buffer but return success and `scratch` returns
@@ -296,9 +337,7 @@ IOStatus RandomAccessFileReader::MultiRead(
   uint64_t elapsed = 0;
   {
     StopWatch sw(clock_, stats_, hist_type_,
-                 (opts.io_activity != Env::IOActivity::kUnknown)
-                     ? kReadHistograms[(std::size_t)(opts.io_activity)]
-                     : Histograms::HISTOGRAM_ENUM_MAX,
+                 GetFileReadHistograms(stats_, opts.io_activity),
                  (stats_ != nullptr) ? &elapsed : nullptr, true /*overwrite*/,
                  true /*delay_enabled*/);
     auto prev_perf_level = GetPerfLevel();
@@ -314,14 +353,14 @@ IOStatus RandomAccessFileReader::MultiRead(
       // Align and merge the read requests.
       size_t alignment = file_->GetRequiredBufferAlignment();
       for (size_t i = 0; i < num_reqs; i++) {
-        const auto& r = Align(read_reqs[i], alignment);
+        FSReadRequest r = Align(read_reqs[i], alignment);
         if (i == 0) {
           // head
-          aligned_reqs.push_back(r);
+          aligned_reqs.push_back(std::move(r));
 
         } else if (!TryMerge(&aligned_reqs.back(), r)) {
           // head + n
-          aligned_reqs.push_back(r);
+          aligned_reqs.push_back(std::move(r));
 
         } else {
           // unused
@@ -487,16 +526,16 @@ IOStatus RandomAccessFileReader::ReadAsync(
 
     assert(read_async_info->buf_.CurrentSize() == 0);
 
-    StopWatch sw(clock_, nullptr /*stats*/,
-                 Histograms::HISTOGRAM_ENUM_MAX /*hist_type*/,
-                 Histograms::HISTOGRAM_ENUM_MAX, &elapsed, true /*overwrite*/,
+    StopWatch sw(clock_, stats_, hist_type_,
+                 GetFileReadHistograms(stats_, opts.io_activity),
+                 (stats_ != nullptr) ? &elapsed : nullptr, true /*overwrite*/,
                  true /*delay_enabled*/);
     s = file_->ReadAsync(aligned_req, opts, read_async_callback,
                          read_async_info, io_handle, del_fn, nullptr /*dbg*/);
   } else {
-    StopWatch sw(clock_, nullptr /*stats*/,
-                 Histograms::HISTOGRAM_ENUM_MAX /*hist_type*/,
-                 Histograms::HISTOGRAM_ENUM_MAX, &elapsed, true /*overwrite*/,
+    StopWatch sw(clock_, stats_, hist_type_,
+                 GetFileReadHistograms(stats_, opts.io_activity),
+                 (stats_ != nullptr) ? &elapsed : nullptr, true /*overwrite*/,
                  true /*delay_enabled*/);
     s = file_->ReadAsync(req, opts, read_async_callback, read_async_info,
                          io_handle, del_fn, nullptr /*dbg*/);

package/deps/rocksdb/rocksdb/file/random_access_file_reader.h

@@ -164,31 +164,18 @@ class RandomAccessFileReader {
   // 2. Otherwise, scratch is not used and can be null, the aligned_buf owns
   // the internally allocated buffer on return, and the result refers to a
   // region in aligned_buf.
-  //
-  // `rate_limiter_priority` is used to charge the internal rate limiter when
-  // enabled. The special value `Env::IO_TOTAL` makes this operation bypass the
-  // rate limiter.
   IOStatus Read(const IOOptions& opts, uint64_t offset, size_t n, Slice* result,
-                char* scratch, AlignedBuf* aligned_buf,
-                Env::IOPriority rate_limiter_priority) const;
+                char* scratch, AlignedBuf* aligned_buf) const;
 
   // REQUIRES:
   // num_reqs > 0, reqs do not overlap, and offsets in reqs are increasing.
   // In non-direct IO mode, aligned_buf should be null;
   // In direct IO mode, aligned_buf stores the aligned buffer allocated inside
   // MultiRead, the result Slices in reqs refer to aligned_buf.
-  //
-  // `rate_limiter_priority` will be used to charge the internal rate limiter.
-  // It is not yet supported so the client must provide the special value
-  // `Env::IO_TOTAL` to bypass the rate limiter.
   IOStatus MultiRead(const IOOptions& opts, FSReadRequest* reqs,
-                     size_t num_reqs, AlignedBuf* aligned_buf,
-                     Env::IOPriority rate_limiter_priority) const;
+                     size_t num_reqs, AlignedBuf* aligned_buf) const;
 
-  IOStatus Prefetch(uint64_t offset, size_t n,
-                    const Env::IOPriority rate_limiter_priority) const {
-    IOOptions opts;
-    opts.rate_limiter_priority = rate_limiter_priority;
+  IOStatus Prefetch(const IOOptions& opts, uint64_t offset, size_t n) const {
     return file_->Prefetch(offset, n, opts, nullptr);
   }
 

package/deps/rocksdb/rocksdb/file/random_access_file_reader_test.cc

@@ -83,8 +83,9 @@ TEST_F(RandomAccessFileReaderTest, ReadDirectIO) {
   Slice result;
   AlignedBuf buf;
   for (Env::IOPriority rate_limiter_priority : {Env::IO_LOW, Env::IO_TOTAL}) {
-    ASSERT_OK(r->Read(IOOptions(), offset, len, &result, nullptr, &buf,
-                      rate_limiter_priority));
+    IOOptions io_opts;
+    io_opts.rate_limiter_priority = rate_limiter_priority;
+    ASSERT_OK(r->Read(io_opts, offset, len, &result, nullptr, &buf));
     ASSERT_EQ(result.ToString(), content.substr(offset, len));
   }
 }
@@ -95,7 +96,18 @@ TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) {
       "RandomAccessFileReader::MultiRead:AlignedReqs", [&](void* reqs) {
         // Copy reqs, since it's allocated on stack inside MultiRead, which will
         // be deallocated after MultiRead returns.
-        aligned_reqs = *reinterpret_cast<std::vector<FSReadRequest>*>(reqs);
+        size_t i = 0;
+        aligned_reqs.resize(
+            (*reinterpret_cast<std::vector<FSReadRequest>*>(reqs)).size());
+        for (auto& req :
+             (*reinterpret_cast<std::vector<FSReadRequest>*>(reqs))) {
+          aligned_reqs[i].offset = req.offset;
+          aligned_reqs[i].len = req.len;
+          aligned_reqs[i].result = req.result;
+          aligned_reqs[i].status = req.status;
+          aligned_reqs[i].scratch = req.scratch;
+          i++;
+        }
       });
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
 
@@ -135,8 +147,8 @@ TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) {
     reqs.push_back(std::move(r0));
     reqs.push_back(std::move(r1));
     AlignedBuf aligned_buf;
-    ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf,
-                           Env::IO_TOTAL /* rate_limiter_priority */));
+    ASSERT_OK(
+        r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf));
 
     AssertResult(content, reqs);
 
@@ -180,8 +192,8 @@ TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) {
     reqs.push_back(std::move(r1));
     reqs.push_back(std::move(r2));
     AlignedBuf aligned_buf;
-    ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf,
-                           Env::IO_TOTAL /* rate_limiter_priority */));
+    ASSERT_OK(
+        r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf));
 
     AssertResult(content, reqs);
 
@@ -225,8 +237,8 @@ TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) {
     reqs.push_back(std::move(r1));
     reqs.push_back(std::move(r2));
     AlignedBuf aligned_buf;
-    ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf,
-                           Env::IO_TOTAL /* rate_limiter_priority */));
+    ASSERT_OK(
+        r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf));
 
     AssertResult(content, reqs);
 
@@ -262,8 +274,8 @@ TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) {
     reqs.push_back(std::move(r0));
     reqs.push_back(std::move(r1));
     AlignedBuf aligned_buf;
-    ASSERT_OK(r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf,
-                           Env::IO_TOTAL /* rate_limiter_priority */));
+    ASSERT_OK(
+        r->MultiRead(IOOptions(), reqs.data(), reqs.size(), &aligned_buf));
 
     AssertResult(content, reqs);
 
@@ -283,7 +295,6 @@ TEST_F(RandomAccessFileReaderTest, MultiReadDirectIO) {
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
 }
 
-
 TEST(FSReadRequest, Align) {
   FSReadRequest r;
   r.offset = 2000;

package/deps/rocksdb/rocksdb/file/sequence_file_reader.h

@@ -99,6 +99,9 @@ class SequentialFileReader {
   // when less than n bytes are actually read (e.g. at end of file). To avoid
   // overcharging the rate limiter, the caller can use file size to cap n to
   // read until end of file.
+  //
+  // TODO(hx235): accept parameter `IOOptions` containing
+  // `rate_limiter_priority` like RandomAccessFileReader::Read()
   IOStatus Read(size_t n, Slice* result, char* scratch,
                 Env::IOPriority rate_limiter_priority);
 

package/deps/rocksdb/rocksdb/include/rocksdb/advanced_cache.h

@@ -13,7 +13,9 @@
 #include <string>
 
 #include "rocksdb/cache.h"
+#include "rocksdb/compression_type.h"
 #include "rocksdb/memory_allocator.h"
+#include "rocksdb/options.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/status.h"
 
@@ -66,7 +68,7 @@ class Cache {
   enum class Priority { HIGH, LOW, BOTTOM };
 
   // A set of callbacks to allow objects in the primary block cache to be
-  // be persisted in a secondary cache. The purpose of the secondary cache
+  // persisted in a secondary cache. The purpose of the secondary cache
   // is to support other ways of caching the object, such as persistent or
   // compressed data, that may require the object to be parsed and transformed
   // in some way. Since the primary cache holds C++ objects and the secondary
@@ -109,13 +111,18 @@ class Cache {
   // pointer into static data).
   using DeleterFn = void (*)(ObjectPtr obj, MemoryAllocator* allocator);
 
-  // The CreateCallback is takes in a buffer from the NVM cache and constructs
-  // an object using it. The callback doesn't have ownership of the buffer and
+  // The CreateCallback is takes in a buffer from the secondary  cache and
+  // constructs an object using it. The buffer could be compressed or
+  // uncompressed, as indicated by the type argument. If compressed,
+  // the callback is responsible for uncompressing it using information
+  // from the context, such as compression dictionary.
+  // The callback doesn't have ownership of the buffer and
   // should copy the contents into its own buffer. The CreateContext* is
   // provided by Lookup and may be used to follow DB- or CF-specific settings.
   // In case of some error, non-OK is returned and the caller should ignore
   // any result in out_obj. (The implementation must clean up after itself.)
-  using CreateCallback = Status (*)(const Slice& data, CreateContext* context,
+  using CreateCallback = Status (*)(const Slice& data, CompressionType type,
+                                    CacheTier source, CreateContext* context,
                                     MemoryAllocator* allocator,
                                     ObjectPtr* out_obj, size_t* out_charge);
 
@@ -242,12 +249,19 @@ class Cache {
   // the item is only inserted into the primary cache. It may
   // defer the insertion to the secondary cache as it sees fit.
   //
+  // Along with the object pointer, the caller may pass a Slice pointing to
+  // the compressed serialized data of the object. If compressed is
+  // non-empty, then the caller must pass the type indicating the compression
+  // algorithm used. The cache may, optionally, also insert the compressed
+  // block into one or more cache tiers.
+  //
   // When the inserted entry is no longer needed, it will be destroyed using
   // helper->del_cb (if non-nullptr).
-  virtual Status Insert(const Slice& key, ObjectPtr obj,
-                        const CacheItemHelper* helper, size_t charge,
-                        Handle** handle = nullptr,
-                        Priority priority = Priority::LOW) = 0;
+  virtual Status Insert(
+      const Slice& key, ObjectPtr obj, const CacheItemHelper* helper,
+      size_t charge, Handle** handle = nullptr,
+      Priority priority = Priority::LOW, const Slice& compressed = Slice(),
+      CompressionType type = CompressionType::kNoCompression) = 0;
 
   // Similar to Insert, but used for creating cache entries that cannot
   // be found with Lookup, such as for memory charging purposes. The
@@ -361,6 +375,14 @@ class Cache {
   // Returns the helper for the specified entry.
   virtual const CacheItemHelper* GetCacheItemHelper(Handle* handle) const = 0;
 
+  virtual Status GetSecondaryCacheCapacity(size_t& /*size*/) const {
+    return Status::NotSupported();
+  }
+
+  virtual Status GetSecondaryCachePinnedUsage(size_t& /*size*/) const {
+    return Status::NotSupported();
+  }
+
   // Call this on shutdown if you want to speed it up. Cache will disown
   // any underlying data and will not free it on delete. This call will leak
   // memory - call this only if you're shutting down the process.
@@ -514,7 +536,8 @@ class Cache {
   // returns `true` if it has taken ownership of the Value (object), or
   // `false` if the cache should destroy it as usual. Regardless, Ref() and
   // Release() cannot be called on this Handle that is poised for eviction.
-  using EvictionCallback = std::function<bool(const Slice& key, Handle* h)>;
+  using EvictionCallback =
+      std::function<bool(const Slice& key, Handle* h, bool was_hit)>;
   // Sets an eviction callback for this Cache. Not thread safe and only
   // supports being set once, so should only be used during initialization
   // or destruction, guaranteed before or after any thread-shared operations.
@@ -535,11 +558,14 @@ class CacheWrapper : public Cache {
   // Only function that derived class must provide
   // const char* Name() const override { ... }
 
-  Status Insert(const Slice& key, ObjectPtr value,
-                const CacheItemHelper* helper, size_t charge,
-                Handle** handle = nullptr,
-                Priority priority = Priority::LOW) override {
-    return target_->Insert(key, value, helper, charge, handle, priority);
+  Status Insert(
+      const Slice& key, ObjectPtr value, const CacheItemHelper* helper,
+      size_t charge, Handle** handle = nullptr,
+      Priority priority = Priority::LOW,
+      const Slice& compressed_value = Slice(),
+      CompressionType type = CompressionType::kNoCompression) override {
+    return target_->Insert(key, value, helper, charge, handle, priority,
+                           compressed_value, type);
   }
 
   Handle* CreateStandalone(const Slice& key, ObjectPtr obj,