@nxtedition/rocksdb 7.1.14 → 7.1.15

package/deps/rocksdb/rocksdb/cache/fast_lru_cache.cc

@@ -173,13 +173,13 @@ inline int LRUHandleTable::FindSlot(const Slice& key,
 LRUCacheShard::LRUCacheShard(size_t capacity, size_t estimated_value_size,
                              bool strict_capacity_limit,
                              CacheMetadataChargePolicy metadata_charge_policy)
-    : capacity_(capacity),
+    : CacheShard(metadata_charge_policy),
+      capacity_(capacity),
       strict_capacity_limit_(strict_capacity_limit),
       table_(
           CalcHashBits(capacity, estimated_value_size, metadata_charge_policy)),
       usage_(0),
       lru_usage_(0) {
-  set_metadata_charge_policy(metadata_charge_policy);
   // Make empty circular linked list.
   lru_.next = &lru_;
   lru_.prev = &lru_;
@@ -525,6 +525,16 @@ size_t LRUCacheShard::GetPinnedUsage() const {
   return usage_ - lru_usage_;
 }
 
+size_t LRUCacheShard::GetOccupancyCount() const {
+  DMutexLock l(mutex_);
+  return table_.GetOccupancy();
+}
+
+size_t LRUCacheShard::GetTableAddressCount() const {
+  DMutexLock l(mutex_);
+  return table_.GetTableSize();
+}
+
 std::string LRUCacheShard::GetPrintableOptions() const { return std::string{}; }
 
 LRUCache::LRUCache(size_t capacity, size_t estimated_value_size,

package/deps/rocksdb/rocksdb/cache/fast_lru_cache.h

@@ -368,6 +368,8 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
 
   size_t GetUsage() const override;
   size_t GetPinnedUsage() const override;
+  size_t GetOccupancyCount() const override;
+  size_t GetTableAddressCount() const override;
 
   void ApplyToSomeEntries(
       const std::function<void(const Slice& key, void* value, size_t charge,

package/deps/rocksdb/rocksdb/cache/lru_cache.cc

@@ -12,6 +12,7 @@
 #include <cassert>
 #include <cstdint>
 #include <cstdio>
+#include <cstdlib>
 
 #include "monitoring/perf_context_imp.h"
 #include "monitoring/statistics.h"
@@ -114,7 +115,8 @@ LRUCacheShard::LRUCacheShard(
     double low_pri_pool_ratio, bool use_adaptive_mutex,
     CacheMetadataChargePolicy metadata_charge_policy, int max_upper_hash_bits,
     const std::shared_ptr<SecondaryCache>& secondary_cache)
-    : capacity_(0),
+    : CacheShard(metadata_charge_policy),
+      capacity_(0),
       high_pri_pool_usage_(0),
       low_pri_pool_usage_(0),
       strict_capacity_limit_(strict_capacity_limit),
@@ -127,7 +129,6 @@ LRUCacheShard::LRUCacheShard(
       lru_usage_(0),
       mutex_(use_adaptive_mutex),
       secondary_cache_(secondary_cache) {
-  set_metadata_charge_policy(metadata_charge_policy);
   // Make empty circular linked list.
   lru_.next = &lru_;
   lru_.prev = &lru_;
@@ -329,6 +330,19 @@ void LRUCacheShard::EvictFromLRU(size_t charge,
   }
 }
 
+void LRUCacheShard::TryInsertIntoSecondaryCache(
+    autovector<LRUHandle*> evicted_handles) {
+  for (auto entry : evicted_handles) {
+    if (secondary_cache_ && entry->IsSecondaryCacheCompatible() &&
+        !entry->IsInSecondaryCache()) {
+      secondary_cache_->Insert(entry->key(), entry->value, entry->info_.helper)
+          .PermitUncheckedError();
+    }
+    // Free the entries here outside of mutex for performance reasons.
+    entry->Free();
+  }
+}
+
 void LRUCacheShard::SetCapacity(size_t capacity) {
   autovector<LRUHandle*> last_reference_list;
   {
@@ -339,16 +353,7 @@ void LRUCacheShard::SetCapacity(size_t capacity) {
     EvictFromLRU(0, &last_reference_list);
   }
 
-  // Try to insert the evicted entries into tiered cache.
-  // Free the entries outside of mutex for performance reasons.
-  for (auto entry : last_reference_list) {
-    if (secondary_cache_ && entry->IsSecondaryCacheCompatible() &&
-        !entry->IsInSecondaryCache()) {
-      secondary_cache_->Insert(entry->key(), entry->value, entry->info_.helper)
-          .PermitUncheckedError();
-    }
-    entry->Free();
-  }
+  TryInsertIntoSecondaryCache(last_reference_list);
 }
 
 void LRUCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
@@ -411,16 +416,7 @@ Status LRUCacheShard::InsertItem(LRUHandle* e, Cache::Handle** handle,
     }
   }
 
-  // Try to insert the evicted entries into the secondary cache.
-  // Free the entries here outside of mutex for performance reasons.
-  for (auto entry : last_reference_list) {
-    if (secondary_cache_ && entry->IsSecondaryCacheCompatible() &&
-        !entry->IsInSecondaryCache()) {
-      secondary_cache_->Insert(entry->key(), entry->value, entry->info_.helper)
-          .PermitUncheckedError();
-    }
-    entry->Free();
-  }
+  TryInsertIntoSecondaryCache(last_reference_list);
 
   return s;
 }
@@ -430,23 +426,70 @@ void LRUCacheShard::Promote(LRUHandle* e) {
 
   assert(secondary_handle->IsReady());
   e->SetIncomplete(false);
-  e->SetInCache(true);
+  e->SetInCache(false);
   e->value = secondary_handle->Value();
   e->CalcTotalCharge(secondary_handle->Size(), metadata_charge_policy_);
   delete secondary_handle;
 
-  // This call could fail if the cache is over capacity and
-  // strict_capacity_limit_ is true. In such a case, we don't want
-  // InsertItem() to free the handle, since the item is already in memory
-  // and the caller will most likely just read from disk if we erase it here.
   if (e->value) {
-    Cache::Handle* handle = reinterpret_cast<Cache::Handle*>(e);
-    Status s = InsertItem(e, &handle, /*free_handle_on_fail=*/false);
+    Status s;
+    if (secondary_cache_ && secondary_cache_->SupportForceErase() &&
+        e->IsStandalone()) {
+      // Insert a dummy handle and return a standalone handle to caller.
+      // Charge the standalone handle.
+      autovector<LRUHandle*> last_reference_list;
+      bool free_standalone_handle{false};
+      {
+        DMutexLock l(mutex_);
+
+        // Free the space following strict LRU policy until enough space
+        // is freed or the lru list is empty.
+        EvictFromLRU(e->total_charge, &last_reference_list);
+
+        if ((usage_ + e->total_charge) > capacity_ && strict_capacity_limit_) {
+          free_standalone_handle = true;
+        } else {
+          usage_ += e->total_charge;
+        }
+      }
+
+      TryInsertIntoSecondaryCache(last_reference_list);
+      if (free_standalone_handle) {
+        e->Unref();
+        e->Free();
+        e = nullptr;
+      } else {
+        PERF_COUNTER_ADD(block_cache_standalone_handle_count, 1);
+      }
+
+      // Insert a dummy handle into the primary cache. This dummy handle is
+      // not IsSecondaryCacheCompatible().
+      Cache::Priority priority =
+          e->IsHighPri() ? Cache::Priority::HIGH : Cache::Priority::LOW;
+      s = Insert(e->key(), e->hash, /*value=*/nullptr, 0,
+                 /*deleter=*/nullptr, /*helper=*/nullptr, /*handle=*/nullptr,
+                 priority);
+    } else {
+      e->SetInCache(true);
+      e->SetIsStandalone(false);
+      Cache::Handle* handle = reinterpret_cast<Cache::Handle*>(e);
+      // This InsertItem() could fail if the cache is over capacity and
+      // strict_capacity_limit_ is true. In such a case, we don't want
+      // InsertItem() to free the handle, since the item is already in memory
+      // and the caller will most likely just read it from disk if we erase it
+      // here.
+      s = InsertItem(e, &handle, /*free_handle_on_fail=*/false);
+      if (s.ok()) {
+        PERF_COUNTER_ADD(block_cache_real_handle_count, 1);
+      }
+    }
+
     if (!s.ok()) {
       // Item is in memory, but not accounted against the cache capacity.
       // When the handle is released, the item should get deleted.
       assert(!e->InCache());
     }
+
   } else {
     // Since the secondary cache lookup failed, mark the item as not in cache
     // Don't charge the cache as its only metadata that'll shortly be released
@@ -454,6 +497,7 @@ void LRUCacheShard::Promote(LRUHandle* e) {
     // TODO
     e->CalcTotalCharge(0, metadata_charge_policy_);
     e->SetInCache(false);
+    e->SetIsStandalone(false);
   }
 }
 
@@ -463,34 +507,59 @@ Cache::Handle* LRUCacheShard::Lookup(
     const ShardedCache::CreateCallback& create_cb, Cache::Priority priority,
     bool wait, Statistics* stats) {
   LRUHandle* e = nullptr;
+  bool found_dummy_entry{false};
   {
     DMutexLock l(mutex_);
     e = table_.Lookup(key, hash);
     if (e != nullptr) {
       assert(e->InCache());
       if (!e->HasRefs()) {
-        // The entry is in LRU since it's in hash and has no external references
+        // The entry is in LRU since it's in hash and has no external
+        // references.
         LRU_Remove(e);
       }
       e->Ref();
       e->SetHit();
+
+      // For a dummy handle, if it was retrieved from secondary cache,
+      // it may still exist in secondary cache.
+      // If the handle exists in secondary cache, the value should be
+      // erased from sec cache and be inserted into primary cache.
+      if (!e->value && secondary_cache_ &&
+          secondary_cache_->SupportForceErase()) {
+        found_dummy_entry = true;
+      }
     }
   }
 
-  // If handle table lookup failed, then allocate a handle outside the
-  // mutex if we're going to lookup in the secondary cache.
+  // If handle table lookup failed or the handle is a dummy one, allocate
+  // a handle outside the mutex if we re going to lookup in the secondary cache.
+  //
+  // When a block is firstly Lookup from CompressedSecondaryCache, we just
+  // insert a dummy block into the primary cache (charging the actual size of
+  // the block) and don't erase the block from CompressedSecondaryCache. A
+  // standalone handle is returned to the caller. Only if the block is hit
+  // again, we erase it from CompressedSecondaryCache and add it into the
+  // primary cache.
+  //
   // Only support synchronous for now.
   // TODO: Support asynchronous lookup in secondary cache
-  if (!e && secondary_cache_ && helper && helper->saveto_cb) {
+  if ((!e || found_dummy_entry) && secondary_cache_ && helper &&
+      helper->saveto_cb) {
     // For objects from the secondary cache, we expect the caller to provide
     // a way to create/delete the primary cache object. The only case where
     // a deleter would not be required is for dummy entries inserted for
     // accounting purposes, which we won't demote to the secondary cache
     // anyway.
     assert(create_cb && helper->del_cb);
+    // Release the dummy handle.
+    if (e) {
+      Release(reinterpret_cast<Cache::Handle*>(e), true /*erase_if_last_ref*/);
+    }
     bool is_in_sec_cache{false};
     std::unique_ptr<SecondaryCacheResultHandle> secondary_handle =
-        secondary_cache_->Lookup(key, create_cb, wait, is_in_sec_cache);
+        secondary_cache_->Lookup(key, create_cb, wait, found_dummy_entry,
+                                 is_in_sec_cache);
     if (secondary_handle != nullptr) {
       e = reinterpret_cast<LRUHandle*>(
           new char[sizeof(LRUHandle) - 1 + key.size()]);
@@ -510,16 +579,22 @@ Cache::Handle* LRUCacheShard::Lookup(
       e->Ref();
       e->SetIsInSecondaryCache(is_in_sec_cache);
 
+      if (secondary_cache_->SupportForceErase() && !found_dummy_entry) {
+        e->SetIsStandalone(true);
+      }
+
       if (wait) {
         Promote(e);
-        if (!e->value) {
-          // The secondary cache returned a handle, but the lookup failed.
-          e->Unref();
-          e->Free();
-          e = nullptr;
-        } else {
-          PERF_COUNTER_ADD(secondary_cache_hit_count, 1);
-          RecordTick(stats, SECONDARY_CACHE_HITS);
+        if (e) {
+          if (!e->value) {
+            // The secondary cache returned a handle, but the lookup failed.
+            e->Unref();
+            e->Free();
+            e = nullptr;
+          } else {
+            PERF_COUNTER_ADD(secondary_cache_hit_count, 1);
+            RecordTick(stats, SECONDARY_CACHE_HITS);
+          }
         }
       } else {
         // If wait is false, we always return a handle and let the caller
@@ -530,6 +605,8 @@ Cache::Handle* LRUCacheShard::Lookup(
         PERF_COUNTER_ADD(secondary_cache_hit_count, 1);
         RecordTick(stats, SECONDARY_CACHE_HITS);
       }
+    } else {
+      e = nullptr;
     }
   }
   return reinterpret_cast<Cache::Handle*>(e);
@@ -682,6 +759,16 @@ size_t LRUCacheShard::GetPinnedUsage() const {
   return usage_ - lru_usage_;
 }
 
+size_t LRUCacheShard::GetOccupancyCount() const {
+  DMutexLock l(mutex_);
+  return table_.GetOccupancyCount();
+}
+
+size_t LRUCacheShard::GetTableAddressCount() const {
+  DMutexLock l(mutex_);
+  return size_t{1} << table_.GetLengthBits();
+}
+
 std::string LRUCacheShard::GetPrintableOptions() const {
   const int kBufferSize = 200;
   char buffer[kBufferSize];
@@ -843,7 +930,7 @@ std::shared_ptr<Cache> NewLRUCache(
     return nullptr;
   }
   if (low_pri_pool_ratio < 0.0 || low_pri_pool_ratio > 1.0) {
-    // Invalid high_pri_pool_ratio
+    // Invalid low_pri_pool_ratio
     return nullptr;
   }
   if (low_pri_pool_ratio + high_pri_pool_ratio > 1.0) {

package/deps/rocksdb/rocksdb/cache/lru_cache.h

@@ -40,8 +40,11 @@ namespace lru_cache {
 //    In that case the entry is not in the LRU list and not in hash table.
 //    The entry can be freed when refs becomes 0.
 //    (refs >= 1 && in_cache == false)
-//
-// All newly created LRUHandles are in state 1. If you call
+// 4. The handle is never inserted into the LRUCache (both hash table and LRU
+//    list) and it doesn't experience the above three states.
+//    The entry can be freed when refs becomes 0.
+//    (refs >= 1 && in_cache == false && IS_STANDALONE == true)
+// All newly created LRUHandles are in state 1 or 4. If you call
 // LRUCacheShard::Release on entry in state 1, it will go into state 2.
 // To move from state 1 to state 3, either call LRUCacheShard::Erase or
 // LRUCacheShard::Insert with the same key (but possibly different value).
@@ -93,6 +96,9 @@ struct LRUHandle {
     IS_LOW_PRI = (1 << 7),
     // Whether this entry is in low-pri pool.
     IN_LOW_PRI_POOL = (1 << 8),
+    // Whether this entry is not inserted into the cache (both hash table and
+    // LRU list).
+    IS_STANDALONE = (1 << 9),
   };
 
   uint16_t flags;
@@ -138,6 +144,7 @@ struct LRUHandle {
   }
   bool IsPending() const { return flags & IS_PENDING; }
   bool IsInSecondaryCache() const { return flags & IS_IN_SECONDARY_CACHE; }
+  bool IsStandalone() const { return flags & IS_STANDALONE; }
 
   void SetInCache(bool in_cache) {
     if (in_cache) {
@@ -205,6 +212,14 @@ struct LRUHandle {
     }
   }
 
+  void SetIsStandalone(bool is_standalone) {
+    if (is_standalone) {
+      flags |= IS_STANDALONE;
+    } else {
+      flags &= ~IS_STANDALONE;
+    }
+  }
+
   void Free() {
     assert(refs == 0);
 #ifdef __SANITIZE_THREAD__
@@ -290,6 +305,8 @@ class LRUHandleTable {
 
   int GetLengthBits() const { return length_bits_; }
 
+  size_t GetOccupancyCount() const { return elems_; }
+
  private:
   // Return a pointer to slot that points to a cache entry that
   // matches key/hash.  If there is no such cache entry, return a
@@ -379,6 +396,8 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
 
   virtual size_t GetUsage() const override;
   virtual size_t GetPinnedUsage() const override;
+  virtual size_t GetOccupancyCount() const override;
+  virtual size_t GetTableAddressCount() const override;
 
   virtual void ApplyToSomeEntries(
       const std::function<void(const Slice& key, void* value, size_t charge,
@@ -435,6 +454,9 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // holding the mutex_.
   void EvictFromLRU(size_t charge, autovector<LRUHandle*>* deleted);
 
+  // Try to insert the evicted handles into the secondary cache.
+  void TryInsertIntoSecondaryCache(autovector<LRUHandle*> evicted_handles);
+
   // Initialized before use.
   size_t capacity_;