@nxtedition/rocksdb 7.1.14 → 7.1.16

package/deps/rocksdb/rocksdb/db/blob/blob_source_test.cc

@@ -13,6 +13,7 @@
 
 #include "cache/charged_cache.h"
 #include "cache/compressed_secondary_cache.h"
+#include "db/blob/blob_contents.h"
 #include "db/blob/blob_file_cache.h"
 #include "db/blob/blob_file_reader.h"
 #include "db/blob/blob_log_format.h"
@@ -218,6 +219,7 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
                                     kNoCompression, prefetch_buffer, &values[i],
                                     &bytes_read));
       ASSERT_EQ(values[i], blobs[i]);
+      ASSERT_TRUE(values[i].IsPinned());
       ASSERT_EQ(bytes_read,
                 BlobLogRecord::kHeaderSize + keys[i].size() + blob_sizes[i]);
 
@@ -255,6 +257,7 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
                                     kNoCompression, prefetch_buffer, &values[i],
                                     &bytes_read));
       ASSERT_EQ(values[i], blobs[i]);
+      ASSERT_TRUE(values[i].IsPinned());
       ASSERT_EQ(bytes_read,
                 BlobLogRecord::kHeaderSize + keys[i].size() + blob_sizes[i]);
 
@@ -298,6 +301,7 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
                                     kNoCompression, prefetch_buffer, &values[i],
                                     &bytes_read));
       ASSERT_EQ(values[i], blobs[i]);
+      ASSERT_TRUE(values[i].IsPinned());
       ASSERT_EQ(bytes_read,
                 BlobLogRecord::kHeaderSize + keys[i].size() + blob_sizes[i]);
 
@@ -336,6 +340,7 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
                                     kNoCompression, prefetch_buffer, &values[i],
                                     &bytes_read));
       ASSERT_EQ(values[i], blobs[i]);
+      ASSERT_TRUE(values[i].IsPinned());
       ASSERT_EQ(bytes_read,
                 BlobLogRecord::kHeaderSize + keys[i].size() + blob_sizes[i]);
 
@@ -382,6 +387,7 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
                                &bytes_read)
                       .IsIncomplete());
       ASSERT_TRUE(values[i].empty());
+      ASSERT_FALSE(values[i].IsPinned());
       ASSERT_EQ(bytes_read, 0);
 
       ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
@@ -423,6 +429,7 @@ TEST_F(BlobSourceTest, GetBlobsFromCache) {
                                &bytes_read)
                       .IsIOError());
       ASSERT_TRUE(values[i].empty());
+      ASSERT_FALSE(values[i].IsPinned());
       ASSERT_EQ(bytes_read, 0);
 
       ASSERT_FALSE(blob_source.TEST_BlobInCache(file_number, file_size,
@@ -855,6 +862,7 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromCache) {
       if (i % 2 == 0) {
         ASSERT_OK(statuses_buf[i]);
         ASSERT_EQ(value_buf[i], blobs[i]);
+        ASSERT_TRUE(value_buf[i].IsPinned());
         fs_read_bytes +=
             blob_sizes[i] + keys[i].size() + BlobLogRecord::kHeaderSize;
         ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
@@ -863,6 +871,7 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromCache) {
       } else {
         statuses_buf[i].PermitUncheckedError();
         ASSERT_TRUE(value_buf[i].empty());
+        ASSERT_FALSE(value_buf[i].IsPinned());
         ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
                                                   blob_offsets[i]));
       }
@@ -895,6 +904,7 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromCache) {
                                     kNoCompression, prefetch_buffer,
                                     &value_buf[i], &bytes_read));
       ASSERT_EQ(value_buf[i], blobs[i]);
+      ASSERT_TRUE(value_buf[i].IsPinned());
       ASSERT_EQ(bytes_read,
                 BlobLogRecord::kHeaderSize + keys[i].size() + blob_sizes[i]);
 
@@ -920,6 +930,7 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromCache) {
     for (size_t i = 0; i < num_blobs; ++i) {
       ASSERT_OK(statuses_buf[i]);
       ASSERT_EQ(value_buf[i], blobs[i]);
+      ASSERT_TRUE(value_buf[i].IsPinned());
       ASSERT_TRUE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
                                                blob_offsets[i]));
       blob_bytes += blob_sizes[i];
@@ -968,6 +979,7 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromCache) {
     for (size_t i = 0; i < num_blobs; ++i) {
       ASSERT_TRUE(statuses_buf[i].IsIncomplete());
       ASSERT_TRUE(value_buf[i].empty());
+      ASSERT_FALSE(value_buf[i].IsPinned());
       ASSERT_FALSE(blob_source.TEST_BlobInCache(blob_file_number, file_size,
                                                 blob_offsets[i]));
     }
@@ -1011,6 +1023,7 @@ TEST_F(BlobSourceTest, MultiGetBlobsFromCache) {
     for (size_t i = 0; i < num_blobs; ++i) {
       ASSERT_TRUE(statuses_buf[i].IsIOError());
       ASSERT_TRUE(value_buf[i].empty());
+      ASSERT_FALSE(value_buf[i].IsPinned());
       ASSERT_FALSE(blob_source.TEST_BlobInCache(non_existing_file_number,
                                                 file_size, blob_offsets[i]));
     }
@@ -1040,16 +1053,17 @@ class BlobSecondaryCacheTest : public DBTestBase {
 
     // Set a small cache capacity to evict entries from the cache, and to test
     // that secondary cache is used properly.
-    lru_cache_ops_.capacity = 1024;
-    lru_cache_ops_.num_shard_bits = 0;
-    lru_cache_ops_.strict_capacity_limit = true;
-    lru_cache_ops_.metadata_charge_policy = kDontChargeCacheMetadata;
-    lru_cache_ops_.high_pri_pool_ratio = 0.2;
-    lru_cache_ops_.low_pri_pool_ratio = 0.2;
+    lru_cache_opts_.capacity = 1024;
+    lru_cache_opts_.num_shard_bits = 0;
+    lru_cache_opts_.strict_capacity_limit = true;
+    lru_cache_opts_.metadata_charge_policy = kDontChargeCacheMetadata;
+    lru_cache_opts_.high_pri_pool_ratio = 0.2;
+    lru_cache_opts_.low_pri_pool_ratio = 0.2;
 
     secondary_cache_opts_.capacity = 8 << 20;  // 8 MB
     secondary_cache_opts_.num_shard_bits = 0;
-    secondary_cache_opts_.metadata_charge_policy = kDontChargeCacheMetadata;
+    secondary_cache_opts_.metadata_charge_policy =
+        kDefaultCacheMetadataChargePolicy;
 
     // Read blobs from the secondary cache if they are not in the primary cache
     options_.lowest_used_cache_tier = CacheTier::kNonVolatileBlockTier;
@@ -1060,7 +1074,7 @@ class BlobSecondaryCacheTest : public DBTestBase {
 
   Options options_;
 
-  LRUCacheOptions lru_cache_ops_;
+  LRUCacheOptions lru_cache_opts_;
   CompressedSecondaryCacheOptions secondary_cache_opts_;
 
   std::string db_id_;
@@ -1073,9 +1087,9 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
   }
 
   secondary_cache_opts_.compression_type = kSnappyCompression;
-  lru_cache_ops_.secondary_cache =
+  lru_cache_opts_.secondary_cache =
       NewCompressedSecondaryCache(secondary_cache_opts_);
-  options_.blob_cache = NewLRUCache(lru_cache_ops_);
+  options_.blob_cache = NewLRUCache(lru_cache_opts_);
 
   options_.cf_paths.emplace_back(
       test::PerThreadDBPath(
@@ -1098,8 +1112,8 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
   Random rnd(301);
 
   std::vector<std::string> key_strs{"key0", "key1"};
-  std::vector<std::string> blob_strs{rnd.RandomString(1010),
-                                     rnd.RandomString(1020)};
+  std::vector<std::string> blob_strs{rnd.RandomString(512),
+                                     rnd.RandomString(768)};
 
   std::vector<Slice> keys{key_strs[0], key_strs[1]};
   std::vector<Slice> blobs{blob_strs[0], blob_strs[1]};
@@ -1134,16 +1148,15 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
   read_options.verify_checksums = true;
 
   auto blob_cache = options_.blob_cache;
-  auto secondary_cache = lru_cache_ops_.secondary_cache;
-
-  Cache::CreateCallback create_cb = [&](const void* buf, size_t size,
-                                        void** out_obj,
-                                        size_t* charge) -> Status {
-    std::string* blob = new std::string();
-    blob->assign(static_cast<const char*>(buf), size);
-    *out_obj = blob;
-    *charge = size;
-    return Status::OK();
+  auto secondary_cache = lru_cache_opts_.secondary_cache;
+
+  Cache::CreateCallback create_cb = [](const void* buf, size_t size,
+                                       void** out_obj,
+                                       size_t* charge) -> Status {
+    CacheAllocationPtr allocation(new char[size]);
+
+    return BlobContents::CreateCallback(std::move(allocation), buf, size,
+                                        out_obj, charge);
   };
 
   {
@@ -1154,6 +1167,25 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
     get_perf_context()->Reset();
 
     // key0 should be filled to the primary cache from the blob file.
+    ASSERT_OK(blob_source.GetBlob(read_options, keys[0], file_number,
+                                  blob_offsets[0], file_size, blob_sizes[0],
+                                  kNoCompression, nullptr /* prefetch_buffer */,
+                                  &values[0], nullptr /* bytes_read */));
+    // Release cache handle
+    values[0].Reset();
+
+    // key0 should be evicted and key0's dummy item is inserted into secondary
+    // cache. key1 should be filled to the primary cache from the blob file.
+    ASSERT_OK(blob_source.GetBlob(read_options, keys[1], file_number,
+                                  blob_offsets[1], file_size, blob_sizes[1],
+                                  kNoCompression, nullptr /* prefetch_buffer */,
+                                  &values[1], nullptr /* bytes_read */));
+
+    // Release cache handle
+    values[1].Reset();
+
+    // key0 should be filled to the primary cache from the blob file. key1
+    // should be evicted and key1's dummy item is inserted into secondary cache.
     ASSERT_OK(blob_source.GetBlob(read_options, keys[0], file_number,
                                   blob_offsets[0], file_size, blob_sizes[0],
                                   kNoCompression, nullptr /* prefetch_buffer */,
@@ -1162,8 +1194,11 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
     ASSERT_TRUE(
         blob_source.TEST_BlobInCache(file_number, file_size, blob_offsets[0]));
 
-    // key0 should be demoted to the secondary cache, and key1 should be filled
-    // to the primary cache from the blob file.
+    // Release cache handle
+    values[0].Reset();
+
+    // key0 should be evicted and is inserted into secondary cache.
+    // key1 should be filled to the primary cache from the blob file.
     ASSERT_OK(blob_source.GetBlob(read_options, keys[1], file_number,
                                   blob_offsets[1], file_size, blob_sizes[1],
                                   kNoCompression, nullptr /* prefetch_buffer */,
@@ -1172,6 +1207,9 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
     ASSERT_TRUE(
         blob_source.TEST_BlobInCache(file_number, file_size, blob_offsets[1]));
 
+    // Release cache handle
+    values[1].Reset();
+
     OffsetableCacheKey base_cache_key(db_id_, db_session_id_, file_number);
 
     // blob_cache here only looks at the primary cache since we didn't provide
@@ -1184,24 +1222,27 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
       auto handle0 = blob_cache->Lookup(key0, statistics);
       ASSERT_EQ(handle0, nullptr);
 
-      // key0 should be in the secondary cache. After looking up key0 in the
-      // secondary cache, it will be erased from the secondary cache.
+      // key0's item should be in the secondary cache.
       bool is_in_sec_cache = false;
       auto sec_handle0 =
-          secondary_cache->Lookup(key0, create_cb, true, is_in_sec_cache);
+          secondary_cache->Lookup(key0, create_cb, true,
+                                  /*advise_erase=*/true, is_in_sec_cache);
       ASSERT_FALSE(is_in_sec_cache);
       ASSERT_NE(sec_handle0, nullptr);
       ASSERT_TRUE(sec_handle0->IsReady());
-      auto value = static_cast<std::string*>(sec_handle0->Value());
-      ASSERT_EQ(*value, blobs[0]);
+      auto value = static_cast<BlobContents*>(sec_handle0->Value());
+      ASSERT_NE(value, nullptr);
+      ASSERT_EQ(value->data(), blobs[0]);
       delete value;
 
-      // key0 doesn't exist in the blob cache
+      // key0 doesn't exist in the blob cache although key0's dummy
+      // item exist in the secondary cache.
       ASSERT_FALSE(blob_source.TEST_BlobInCache(file_number, file_size,
                                                 blob_offsets[0]));
     }
 
-    // key1 should exist in the primary cache.
+    // key1 should exists in the primary cache. key1's dummy item exists
+    // in the secondary cache.
     {
       CacheKey cache_key = base_cache_key.WithOffset(blob_offsets[1]);
       const Slice key1 = cache_key.AsSlice();
@@ -1211,7 +1252,8 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
 
       bool is_in_sec_cache = false;
       auto sec_handle1 =
-          secondary_cache->Lookup(key1, create_cb, true, is_in_sec_cache);
+          secondary_cache->Lookup(key1, create_cb, true,
+                                  /*advise_erase=*/true, is_in_sec_cache);
       ASSERT_FALSE(is_in_sec_cache);
       ASSERT_EQ(sec_handle1, nullptr);
 
@@ -1221,23 +1263,27 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
 
     {
       // fetch key0 from the blob file to the primary cache.
+      // key1 is evicted and inserted into the secondary cache.
       ASSERT_OK(blob_source.GetBlob(
           read_options, keys[0], file_number, blob_offsets[0], file_size,
           blob_sizes[0], kNoCompression, nullptr /* prefetch_buffer */,
           &values[0], nullptr /* bytes_read */));
       ASSERT_EQ(values[0], blobs[0]);
 
+      // Release cache handle
+      values[0].Reset();
+
       // key0 should be in the primary cache.
       CacheKey cache_key0 = base_cache_key.WithOffset(blob_offsets[0]);
       const Slice key0 = cache_key0.AsSlice();
       auto handle0 = blob_cache->Lookup(key0, statistics);
       ASSERT_NE(handle0, nullptr);
-      auto value = static_cast<std::string*>(blob_cache->Value(handle0));
-      ASSERT_EQ(*value, blobs[0]);
+      auto value = static_cast<BlobContents*>(blob_cache->Value(handle0));
+      ASSERT_NE(value, nullptr);
+      ASSERT_EQ(value->data(), blobs[0]);
       blob_cache->Release(handle0);
 
-      // key1 is not in the primary cache, and it should be demoted to the
-      // secondary cache.
+      // key1 is not in the primary cache and is in the secondary cache.
       CacheKey cache_key1 = base_cache_key.WithOffset(blob_offsets[1]);
       const Slice key1 = cache_key1.AsSlice();
       auto handle1 = blob_cache->Lookup(key1, statistics);
@@ -1251,16 +1297,19 @@ TEST_F(BlobSecondaryCacheTest, GetBlobsFromSecondaryCache) {
       // key1 promotion should succeed due to the primary cache being empty. we
       // did't call secondary cache's Lookup() here, because it will remove the
       // key but it won't be able to promote the key to the primary cache.
-      // Instead we use the end-to-end blob source API to promote the key to
-      // the primary cache.
+      // Instead we use the end-to-end blob source API to read key1.
+      // In function TEST_BlobInCache, key1's dummy item is inserted into the
+      // primary cache and a standalone handle is checked by GetValue().
       ASSERT_TRUE(blob_source.TEST_BlobInCache(file_number, file_size,
                                                blob_offsets[1]));
 
-      // key1 should be in the primary cache.
+      // key1's dummy handle is in the primary cache and key1's item is still
+      // in the secondary cache. So, the primary cache's Lookup() can only
+      // get a dummy handle.
       handle1 = blob_cache->Lookup(key1, statistics);
       ASSERT_NE(handle1, nullptr);
-      value = static_cast<std::string*>(blob_cache->Value(handle1));
-      ASSERT_EQ(*value, blobs[1]);
+      // handl1 is a dummy handle.
+      ASSERT_EQ(blob_cache->Value(handle1), nullptr);
       blob_cache->Release(handle1);
     }
   }
@@ -1385,11 +1434,11 @@ TEST_F(BlobSourceCacheReservationTest, SimpleCacheReservation) {
   ReadOptions read_options;
   read_options.verify_checksums = true;
 
-  std::vector<PinnableSlice> values(keys_.size());
-
   {
     read_options.fill_cache = false;
 
+    std::vector<PinnableSlice> values(keys_.size());
+
     for (size_t i = 0; i < kNumBlobs; ++i) {
       ASSERT_OK(blob_source.GetBlob(
           read_options, keys_[i], kBlobFileNumber, blob_offsets[i],
@@ -1403,6 +1452,8 @@ TEST_F(BlobSourceCacheReservationTest, SimpleCacheReservation) {
   {
     read_options.fill_cache = true;
 
+    std::vector<PinnableSlice> values(keys_.size());
+
     // num_blobs is 16, so the total blob cache usage is less than a single
     // dummy entry. Therefore, cache reservation manager only reserves one dummy
     // entry here.
@@ -1412,7 +1463,12 @@ TEST_F(BlobSourceCacheReservationTest, SimpleCacheReservation) {
           read_options, keys_[i], kBlobFileNumber, blob_offsets[i],
           blob_file_size_, blob_sizes[i], kNoCompression,
           nullptr /* prefetch_buffer */, &values[i], nullptr /* bytes_read */));
-      blob_bytes += blob_sizes[i];
+
+      size_t charge = 0;
+      ASSERT_TRUE(blob_source.TEST_BlobInCache(kBlobFileNumber, blob_file_size_,
+                                               blob_offsets[i], &charge));
+
+      blob_bytes += charge;
       ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), kSizeDummyEntry);
       ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), blob_bytes);
       ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(),
@@ -1425,19 +1481,23 @@ TEST_F(BlobSourceCacheReservationTest, SimpleCacheReservation) {
     size_t blob_bytes = options_.blob_cache->GetUsage();
 
     for (size_t i = 0; i < kNumBlobs; ++i) {
+      size_t charge = 0;
+      ASSERT_TRUE(blob_source.TEST_BlobInCache(kBlobFileNumber, blob_file_size_,
+                                               blob_offsets[i], &charge));
+
       CacheKey cache_key = base_cache_key.WithOffset(blob_offsets[i]);
       // We didn't call options_.blob_cache->Erase() here, this is because
       // the cache wrapper's Erase() method must be called to update the
       // cache usage after erasing the cache entry.
       blob_source.GetBlobCache()->Erase(cache_key.AsSlice());
       if (i == kNumBlobs - 1) {
-        // The last blob is not in the cache. cache_res_mgr should not reserve
-        // any space for it.
+        // All the blobs got removed from the cache. cache_res_mgr should not
+        // reserve any space for them.
         ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), 0);
       } else {
         ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), kSizeDummyEntry);
       }
-      blob_bytes -= blob_sizes[i];
+      blob_bytes -= charge;
       ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), blob_bytes);
       ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(),
                 options_.blob_cache->GetUsage());
@@ -1495,11 +1555,11 @@ TEST_F(BlobSourceCacheReservationTest, IncreaseCacheReservationOnFullCache) {
   ReadOptions read_options;
   read_options.verify_checksums = true;
 
-  std::vector<PinnableSlice> values(keys_.size());
-
   {
     read_options.fill_cache = false;
 
+    std::vector<PinnableSlice> values(keys_.size());
+
     for (size_t i = 0; i < kNumBlobs; ++i) {
       ASSERT_OK(blob_source.GetBlob(
           read_options, keys_[i], kBlobFileNumber, blob_offsets[i],
@@ -1513,21 +1573,33 @@ TEST_F(BlobSourceCacheReservationTest, IncreaseCacheReservationOnFullCache) {
   {
     read_options.fill_cache = true;
 
-    // Since we resized each blob to be kSizeDummyEntry / (num_blobs/ 2), we
-    // should observe cache eviction for the second half blobs.
+    std::vector<PinnableSlice> values(keys_.size());
+
+    // Since we resized each blob to be kSizeDummyEntry / (num_blobs / 2), we
+    // can't fit all the blobs in the cache at the same time, which means we
+    // should observe cache evictions once we reach the cache's capacity.
+    // Due to the overhead of the cache and the BlobContents objects, as well as
+    // jemalloc bin sizes, this happens after inserting seven blobs.
     uint64_t blob_bytes = 0;
     for (size_t i = 0; i < kNumBlobs; ++i) {
       ASSERT_OK(blob_source.GetBlob(
           read_options, keys_[i], kBlobFileNumber, blob_offsets[i],
           blob_file_size_, blob_sizes[i], kNoCompression,
           nullptr /* prefetch_buffer */, &values[i], nullptr /* bytes_read */));
-      blob_bytes += blob_sizes[i];
-      ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), kSizeDummyEntry);
-      if (i >= kNumBlobs / 2) {
-        ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), kSizeDummyEntry);
-      } else {
-        ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), blob_bytes);
+
+      // Release cache handle
+      values[i].Reset();
+
+      if (i < kNumBlobs / 2 - 1) {
+        size_t charge = 0;
+        ASSERT_TRUE(blob_source.TEST_BlobInCache(
+            kBlobFileNumber, blob_file_size_, blob_offsets[i], &charge));
+
+        blob_bytes += charge;
       }
+
+      ASSERT_EQ(cache_res_mgr->GetTotalReservedCacheSize(), kSizeDummyEntry);
+      ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(), blob_bytes);
       ASSERT_EQ(cache_res_mgr->GetTotalMemoryUsed(),
                 options_.blob_cache->GetUsage());
     }

package/deps/rocksdb/rocksdb/db/blob/db_blob_basic_test.cc

@@ -7,6 +7,7 @@
 #include <sstream>
 #include <string>
 
+#include "cache/compressed_secondary_cache.h"
 #include "db/blob/blob_index.h"
 #include "db/blob/blob_log_format.h"
 #include "db/db_test_util.h"
@@ -53,7 +54,7 @@ TEST_F(DBBlobBasicTest, GetBlobFromCache) {
   Options options = GetDefaultOptions();
 
   LRUCacheOptions co;
-  co.capacity = 2048;
+  co.capacity = 2 << 20;  // 2MB
   co.num_shard_bits = 2;
   co.metadata_charge_policy = kDontChargeCacheMetadata;
   auto backing_cache = NewLRUCache(co);
@@ -121,7 +122,7 @@ TEST_F(DBBlobBasicTest, IterateBlobsFromCache) {
   Options options = GetDefaultOptions();
 
   LRUCacheOptions co;
-  co.capacity = 2048;
+  co.capacity = 2 << 20;  // 2MB
   co.num_shard_bits = 2;
   co.metadata_charge_policy = kDontChargeCacheMetadata;
   auto backing_cache = NewLRUCache(co);
@@ -452,7 +453,7 @@ TEST_F(DBBlobBasicTest, MultiGetBlobsFromCache) {
   Options options = GetDefaultOptions();
 
   LRUCacheOptions co;
-  co.capacity = 2048;
+  co.capacity = 2 << 20;  // 2MB
   co.num_shard_bits = 2;
   co.metadata_charge_policy = kDontChargeCacheMetadata;
   auto backing_cache = NewLRUCache(co);
@@ -1701,6 +1702,83 @@ TEST_F(DBBlobBasicTest, DynamicallyWarmCacheDuringFlush) {
 }
 #endif  // !ROCKSDB_LITE
 
+TEST_F(DBBlobBasicTest, WarmCacheWithBlobsSecondary) {
+  CompressedSecondaryCacheOptions secondary_cache_opts;
+  secondary_cache_opts.capacity = 1 << 20;
+  secondary_cache_opts.num_shard_bits = 0;
+  secondary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
+  secondary_cache_opts.compression_type = kNoCompression;
+
+  LRUCacheOptions primary_cache_opts;
+  primary_cache_opts.capacity = 1024;
+  primary_cache_opts.num_shard_bits = 0;
+  primary_cache_opts.metadata_charge_policy = kDontChargeCacheMetadata;
+  primary_cache_opts.secondary_cache =
+      NewCompressedSecondaryCache(secondary_cache_opts);
+
+  Options options = GetDefaultOptions();
+  options.create_if_missing = true;
+  options.statistics = CreateDBStatistics();
+  options.enable_blob_files = true;
+  options.blob_cache = NewLRUCache(primary_cache_opts);
+  options.prepopulate_blob_cache = PrepopulateBlobCache::kFlushOnly;
+
+  DestroyAndReopen(options);
+
+  // Note: only one of the two blobs fit in the primary cache at any given time.
+  constexpr char first_key[] = "foo";
+  constexpr size_t first_blob_size = 512;
+  const std::string first_blob(first_blob_size, 'a');
+
+  constexpr char second_key[] = "bar";
+  constexpr size_t second_blob_size = 768;
+  const std::string second_blob(second_blob_size, 'b');
+
+  // First blob is inserted into primary cache during flush.
+  ASSERT_OK(Put(first_key, first_blob));
+  ASSERT_OK(Flush());
+  ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), 1);
+
+  // Second blob is inserted into primary cache during flush,
+  // First blob is evicted but only a dummy handle is inserted into secondary
+  // cache.
+  ASSERT_OK(Put(second_key, second_blob));
+  ASSERT_OK(Flush());
+  ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_ADD), 1);
+
+  // First blob is inserted into primary cache.
+  // Second blob is evicted but only a dummy handle is inserted into secondary
+  // cache.
+  ASSERT_EQ(Get(first_key), first_blob);
+  ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_MISS), 1);
+  ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_HIT), 0);
+  ASSERT_EQ(options.statistics->getAndResetTickerCount(SECONDARY_CACHE_HITS),
+            0);
+  // Second blob is inserted into primary cache,
+  // First blob is evicted and is inserted into secondary cache.
+  ASSERT_EQ(Get(second_key), second_blob);
+  ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_MISS), 1);
+  ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_HIT), 0);
+  ASSERT_EQ(options.statistics->getAndResetTickerCount(SECONDARY_CACHE_HITS),
+            0);
+
+  // First blob's dummy item is inserted into primary cache b/c of lookup.
+  // Second blob is still in primary cache.
+  ASSERT_EQ(Get(first_key), first_blob);
+  ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_MISS), 0);
+  ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_HIT), 1);
+  ASSERT_EQ(options.statistics->getAndResetTickerCount(SECONDARY_CACHE_HITS),
+            1);
+
+  // First blob's item is inserted into primary cache b/c of lookup.
+  // Second blob is evicted and inserted into secondary cache.
+  ASSERT_EQ(Get(first_key), first_blob);
+  ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_MISS), 0);
+  ASSERT_EQ(options.statistics->getAndResetTickerCount(BLOB_DB_CACHE_HIT), 1);
+  ASSERT_EQ(options.statistics->getAndResetTickerCount(SECONDARY_CACHE_HITS),
+            1);
+}
+
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

package/deps/rocksdb/rocksdb/db/c.cc

@@ -1054,6 +1054,18 @@ void rocksdb_drop_column_family(
   SaveError(errptr, db->rep->DropColumnFamily(handle->rep));
 }
 
+uint32_t rocksdb_column_family_handle_get_id(
+    rocksdb_column_family_handle_t* handle) {
+  return handle->rep->GetID();
+}
+
+char* rocksdb_column_family_handle_get_name(
+    rocksdb_column_family_handle_t* handle, size_t* name_len) {
+  auto name = handle->rep->GetName();
+  *name_len = name.size();
+  return CopyString(name);
+}
+
 void rocksdb_column_family_handle_destroy(rocksdb_column_family_handle_t* handle) {
   delete handle->rep;
   delete handle;
@@ -2606,6 +2618,11 @@ void rocksdb_block_based_options_destroy(
   delete options;
 }
 
+void rocksdb_block_based_options_set_checksum(
+    rocksdb_block_based_table_options_t* opt, char v) {
+  opt->rep.checksum = static_cast<ROCKSDB_NAMESPACE::ChecksumType>(v);
+}
+
 void rocksdb_block_based_options_set_block_size(
     rocksdb_block_based_table_options_t* options, size_t block_size) {
   options->rep.block_size = block_size;
@@ -4099,6 +4116,8 @@ uint64_t rocksdb_perfcontext_metric(rocksdb_perfcontext_t* context,
       return rep->blob_checksum_time;
     case rocksdb_blob_decompress_time:
       return rep->blob_decompress_time;
+    case rocksdb_internal_range_del_reseek_count:
+      return rep->internal_range_del_reseek_count;
     default:
       break;
   }
@@ -6449,6 +6468,16 @@ void rocksdb_options_set_memtable_whole_key_filtering(rocksdb_options_t* opt,
   opt->rep.memtable_whole_key_filtering = val;
 }
 
+void rocksdb_options_set_avoid_unnecessary_blocking_io(rocksdb_options_t* opt,
+                                                       unsigned char val) {
+  opt->rep.avoid_unnecessary_blocking_io = val;
+}
+
+unsigned char rocksdb_options_get_avoid_unnecessary_blocking_io(
+    rocksdb_options_t* opt) {
+  return opt->rep.avoid_unnecessary_blocking_io;
+}
+
 // deletes container with memory usage estimates
 void rocksdb_approximate_memory_usage_destroy(rocksdb_memory_usage_t* usage) {
   delete usage;

package/deps/rocksdb/rocksdb/db/column_family.cc

@@ -1144,7 +1144,8 @@ Status ColumnFamilyData::RangesOverlapWithMemtables(
   MergeIteratorBuilder merge_iter_builder(&internal_comparator_, &arena);
   merge_iter_builder.AddIterator(
       super_version->mem->NewIterator(read_opts, &arena));
-  super_version->imm->AddIterators(read_opts, &merge_iter_builder);
+  super_version->imm->AddIterators(read_opts, &merge_iter_builder,
+                                   false /* add_range_tombstone_iter */);
   ScopedArenaIterator memtable_iter(merge_iter_builder.Finish());
 
   auto read_seq = super_version->current->version_set()->LastSequence();
@@ -1415,6 +1416,14 @@ Status ColumnFamilyData::ValidateOptions(
         "FIFO compaction only supported with max_open_files = -1.");
   }
 
+  std::vector<uint32_t> supported{0, 1, 2, 4, 8};
+  if (std::find(supported.begin(), supported.end(),
+                cf_options.memtable_protection_bytes_per_key) ==
+      supported.end()) {
+    return Status::NotSupported(
+        "Memtable per key-value checksum protection only supports 0, 1, 2, 4 "
+        "or 8 bytes per key.");
+  }
   return s;
 }