@nxtedition/rocksdb 8.1.17 → 8.2.0-alpha.2

package/deps/rocksdb/rocksdb/cache/lru_cache.cc

@@ -14,6 +14,7 @@
 #include <cstdio>
 #include <cstdlib>
 
+#include "cache/secondary_cache_adapter.h"
 #include "monitoring/perf_context_imp.h"
 #include "monitoring/statistics.h"
 #include "port/lang.h"
@@ -22,14 +23,6 @@
 namespace ROCKSDB_NAMESPACE {
 namespace lru_cache {
 
-namespace {
-// A distinct pointer value for marking "dummy" cache entries
-struct DummyValue {
-  char val[12] = "kDummyValue";
-};
-DummyValue kDummyValue{};
-}  // namespace
-
 LRUHandleTable::LRUHandleTable(int max_upper_hash_bits,
                                MemoryAllocator* allocator)
     : length_bits_(/* historical starting size*/ 4),
@@ -103,7 +96,7 @@ void LRUHandleTable::Resize() {
   std::unique_ptr<LRUHandle* []> new_list {
     new LRUHandle* [size_t{1} << new_length_bits] {}
   };
-  uint32_t count = 0;
+  [[maybe_unused]] uint32_t count = 0;
   for (uint32_t i = 0; i < old_length; i++) {
     LRUHandle* h = list_[i];
     while (h != nullptr) {
@@ -127,7 +120,7 @@ LRUCacheShard::LRUCacheShard(size_t capacity, bool strict_capacity_limit,
                              CacheMetadataChargePolicy metadata_charge_policy,
                              int max_upper_hash_bits,
                              MemoryAllocator* allocator,
-                             SecondaryCache* secondary_cache)
+                             const Cache::EvictionCallback* eviction_callback)
     : CacheShardBase(metadata_charge_policy),
       capacity_(0),
       high_pri_pool_usage_(0),
@@ -141,7 +134,7 @@ LRUCacheShard::LRUCacheShard(size_t capacity, bool strict_capacity_limit,
       usage_(0),
       lru_usage_(0),
       mutex_(use_adaptive_mutex),
-      secondary_cache_(secondary_cache) {
+      eviction_callback_(*eviction_callback) {
   // Make empty circular linked list.
   lru_.next = &lru_;
   lru_.prev = &lru_;
@@ -341,16 +334,19 @@ void LRUCacheShard::EvictFromLRU(size_t charge,
   }
 }
 
-void LRUCacheShard::TryInsertIntoSecondaryCache(
-    autovector<LRUHandle*> evicted_handles) {
-  for (auto entry : evicted_handles) {
-    if (secondary_cache_ && entry->IsSecondaryCacheCompatible() &&
-        !entry->IsInSecondaryCache()) {
-      secondary_cache_->Insert(entry->key(), entry->value, entry->helper)
-          .PermitUncheckedError();
+void LRUCacheShard::NotifyEvicted(
+    const autovector<LRUHandle*>& evicted_handles) {
+  MemoryAllocator* alloc = table_.GetAllocator();
+  for (LRUHandle* entry : evicted_handles) {
+    if (eviction_callback_ &&
+        eviction_callback_(entry->key(),
+                           reinterpret_cast<Cache::Handle*>(entry))) {
+      // Callback took ownership of obj; just free handle
+      free(entry);
+    } else {
+      // Free the entries here outside of mutex for performance reasons.
+      entry->Free(alloc);
     }
-    // Free the entries here outside of mutex for performance reasons.
-    entry->Free(table_.GetAllocator());
   }
 }
 
@@ -364,7 +360,7 @@ void LRUCacheShard::SetCapacity(size_t capacity) {
     EvictFromLRU(0, &last_reference_list);
   }
 
-  TryInsertIntoSecondaryCache(last_reference_list);
+  NotifyEvicted(last_reference_list);
 }
 
 void LRUCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
@@ -372,8 +368,7 @@ void LRUCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
   strict_capacity_limit_ = strict_capacity_limit;
 }
 
-Status LRUCacheShard::InsertItem(LRUHandle* e, LRUHandle** handle,
-                                 bool free_handle_on_fail) {
+Status LRUCacheShard::InsertItem(LRUHandle* e, LRUHandle** handle) {
   Status s = Status::OK();
   autovector<LRUHandle*> last_reference_list;
 
@@ -392,10 +387,9 @@ Status LRUCacheShard::InsertItem(LRUHandle* e, LRUHandle** handle,
         // into cache and get evicted immediately.
         last_reference_list.push_back(e);
       } else {
-        if (free_handle_on_fail) {
-          free(e);
-          *handle = nullptr;
-        }
+        free(e);
+        e = nullptr;
+        *handle = nullptr;
         s = Status::MemoryLimit("Insert failed due to LRU cache being full.");
       }
     } else {
@@ -427,185 +421,27 @@ Status LRUCacheShard::InsertItem(LRUHandle* e, LRUHandle** handle,
     }
   }
 
-  TryInsertIntoSecondaryCache(last_reference_list);
+  NotifyEvicted(last_reference_list);
 
   return s;
 }
 
-void LRUCacheShard::Promote(LRUHandle* e) {
-  SecondaryCacheResultHandle* secondary_handle = e->sec_handle;
-
-  assert(secondary_handle->IsReady());
-  // e is not thread-shared here; OK to modify "immutable" fields as well as
-  // "mutable" (normally requiring mutex)
-  e->SetIsPending(false);
-  e->value = secondary_handle->Value();
-  assert(e->total_charge == 0);
-  size_t value_size = secondary_handle->Size();
-  delete secondary_handle;
-
-  if (e->value) {
-    e->CalcTotalCharge(value_size, metadata_charge_policy_);
-    Status s;
-    if (e->IsStandalone()) {
-      assert(secondary_cache_ && secondary_cache_->SupportForceErase());
-
-      // Insert a dummy handle and return a standalone handle to caller.
-      // Charge the standalone handle.
-      autovector<LRUHandle*> last_reference_list;
-      bool free_standalone_handle{false};
-      {
-        DMutexLock l(mutex_);
-
-        // Free the space following strict LRU policy until enough space
-        // is freed or the lru list is empty.
-        EvictFromLRU(e->total_charge, &last_reference_list);
-
-        if ((usage_ + e->total_charge) > capacity_ && strict_capacity_limit_) {
-          free_standalone_handle = true;
-        } else {
-          usage_ += e->total_charge;
-        }
-      }
-
-      TryInsertIntoSecondaryCache(last_reference_list);
-      if (free_standalone_handle) {
-        e->Unref();
-        e->Free(table_.GetAllocator());
-        e = nullptr;
-      } else {
-        PERF_COUNTER_ADD(block_cache_standalone_handle_count, 1);
-      }
-
-      // Insert a dummy handle into the primary cache. This dummy handle is
-      // not IsSecondaryCacheCompatible().
-      // FIXME? This should not overwrite an existing non-dummy entry in the
-      // rare case that one exists
-      Cache::Priority priority =
-          e->IsHighPri() ? Cache::Priority::HIGH : Cache::Priority::LOW;
-      s = Insert(e->key(), e->hash, &kDummyValue, &kNoopCacheItemHelper,
-                 /*charge=*/0,
-                 /*handle=*/nullptr, priority);
-    } else {
-      e->SetInCache(true);
-      LRUHandle* handle = e;
-      // This InsertItem() could fail if the cache is over capacity and
-      // strict_capacity_limit_ is true. In such a case, we don't want
-      // InsertItem() to free the handle, since the item is already in memory
-      // and the caller will most likely just read it from disk if we erase it
-      // here.
-      s = InsertItem(e, &handle, /*free_handle_on_fail=*/false);
-      if (s.ok()) {
-        PERF_COUNTER_ADD(block_cache_real_handle_count, 1);
-      }
-    }
-
-    if (!s.ok()) {
-      // Item is in memory, but not accounted against the cache capacity.
-      // When the handle is released, the item should get deleted.
-      assert(!e->InCache());
-    }
-  } else {
-    // Secondary cache lookup failed. The caller will take care of detecting
-    // this and eventually releasing e.
-    assert(!e->value);
-    assert(!e->InCache());
-  }
-}
-
 LRUHandle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash,
-                                 const Cache::CacheItemHelper* helper,
-                                 Cache::CreateContext* create_context,
-                                 Cache::Priority priority, bool wait,
-                                 Statistics* stats) {
-  LRUHandle* e = nullptr;
-  bool found_dummy_entry{false};
-  {
-    DMutexLock l(mutex_);
-    e = table_.Lookup(key, hash);
-    if (e != nullptr) {
-      assert(e->InCache());
-      if (e->value == &kDummyValue) {
-        // For a dummy handle, if it was retrieved from secondary cache,
-        // it may still exist in secondary cache.
-        // If the handle exists in secondary cache, the value should be
-        // erased from sec cache and be inserted into primary cache.
-        found_dummy_entry = true;
-        // Let the dummy entry be overwritten
-        e = nullptr;
-      } else {
-        if (!e->HasRefs()) {
-          // The entry is in LRU since it's in hash and has no external
-          // references.
-          LRU_Remove(e);
-        }
-        e->Ref();
-        e->SetHit();
-      }
-    }
-  }
-
-  // If handle table lookup failed or the handle is a dummy one, allocate
-  // a handle outside the mutex if we re going to lookup in the secondary cache.
-  //
-  // When a block is firstly Lookup from CompressedSecondaryCache, we just
-  // insert a dummy block into the primary cache (charging the actual size of
-  // the block) and don't erase the block from CompressedSecondaryCache. A
-  // standalone handle is returned to the caller. Only if the block is hit
-  // again, we erase it from CompressedSecondaryCache and add it into the
-  // primary cache.
-  if (!e && secondary_cache_ && helper && helper->create_cb) {
-    bool is_in_sec_cache{false};
-    std::unique_ptr<SecondaryCacheResultHandle> secondary_handle =
-        secondary_cache_->Lookup(key, helper, create_context, wait,
-                                 found_dummy_entry, is_in_sec_cache);
-    if (secondary_handle != nullptr) {
-      e = static_cast<LRUHandle*>(malloc(sizeof(LRUHandle) - 1 + key.size()));
-
-      e->m_flags = 0;
-      e->im_flags = 0;
-      e->helper = helper;
-      e->key_length = key.size();
-      e->hash = hash;
-      e->refs = 0;
-      e->next = e->prev = nullptr;
-      e->SetPriority(priority);
-      memcpy(e->key_data, key.data(), key.size());
-      e->value = nullptr;
-      e->sec_handle = secondary_handle.release();
-      e->total_charge = 0;
-      e->Ref();
-      e->SetIsInSecondaryCache(is_in_sec_cache);
-      e->SetIsStandalone(secondary_cache_->SupportForceErase() &&
-                         !found_dummy_entry);
-
-      if (wait) {
-        Promote(e);
-        if (e) {
-          if (!e->value) {
-            // The secondary cache returned a handle, but the lookup failed.
-            e->Unref();
-            e->Free(table_.GetAllocator());
-            e = nullptr;
-          } else {
-            PERF_COUNTER_ADD(secondary_cache_hit_count, 1);
-            RecordTick(stats, SECONDARY_CACHE_HITS);
-          }
-        }
-      } else {
-        // If wait is false, we always return a handle and let the caller
-        // release the handle after checking for success or failure.
-        e->SetIsPending(true);
-        // This may be slightly inaccurate, if the lookup eventually fails.
-        // But the probability is very low.
-        PERF_COUNTER_ADD(secondary_cache_hit_count, 1);
-        RecordTick(stats, SECONDARY_CACHE_HITS);
-      }
-    } else {
-      // Caller will most likely overwrite the dummy entry with an Insert
-      // after this Lookup fails
-      assert(e == nullptr);
+                                 const Cache::CacheItemHelper* /*helper*/,
+                                 Cache::CreateContext* /*create_context*/,
+                                 Cache::Priority /*priority*/,
+                                 Statistics* /*stats*/) {
+  DMutexLock l(mutex_);
+  LRUHandle* e = table_.Lookup(key, hash);
+  if (e != nullptr) {
+    assert(e->InCache());
+    if (!e->HasRefs()) {
+      // The entry is in LRU since it's in hash and has no external
+      // references.
+      LRU_Remove(e);
     }
+    e->Ref();
+    e->SetHit();
   }
   return e;
 }
@@ -614,8 +450,6 @@ bool LRUCacheShard::Ref(LRUHandle* e) {
   DMutexLock l(mutex_);
   // To create another reference - entry must be already externally referenced.
   assert(e->HasRefs());
-  // Pending handles are not for sharing
-  assert(!e->IsPending());
   e->Ref();
   return true;
 }
@@ -639,14 +473,13 @@ bool LRUCacheShard::Release(LRUHandle* e, bool /*useful*/,
   if (e == nullptr) {
     return false;
   }
-  bool last_reference = false;
-  // Must Wait or WaitAll first on pending handles. Otherwise, would leak
-  // a secondary cache handle.
-  assert(!e->IsPending());
+  bool must_free;
+  bool was_in_cache;
   {
     DMutexLock l(mutex_);
-    last_reference = e->Unref();
-    if (last_reference && e->InCache()) {
+    must_free = e->Unref();
+    was_in_cache = e->InCache();
+    if (must_free && was_in_cache) {
       // The item is still in cache, and nobody else holds a reference to it.
       if (usage_ > capacity_ || erase_if_last_ref) {
         // The LRU list must be empty since the cache is full.
@@ -657,29 +490,38 @@ bool LRUCacheShard::Release(LRUHandle* e, bool /*useful*/,
       } else {
         // Put the item back on the LRU list, and don't free it.
         LRU_Insert(e);
-        last_reference = false;
+        must_free = false;
       }
     }
-    // If it was the last reference, then decrement the cache usage.
-    if (last_reference) {
+    // If about to be freed, then decrement the cache usage.
+    if (must_free) {
       assert(usage_ >= e->total_charge);
       usage_ -= e->total_charge;
     }
   }
 
   // Free the entry here outside of mutex for performance reasons.
-  if (last_reference) {
-    e->Free(table_.GetAllocator());
+  if (must_free) {
+    // Only call eviction callback if we're sure no one requested erasure
+    // FIXME: disabled because of test churn
+    if (false && was_in_cache && !erase_if_last_ref && eviction_callback_ &&
+        eviction_callback_(e->key(), reinterpret_cast<Cache::Handle*>(e))) {
+      // Callback took ownership of obj; just free handle
+      free(e);
+    } else {
+      e->Free(table_.GetAllocator());
+    }
   }
-  return last_reference;
+  return must_free;
 }
 
-Status LRUCacheShard::Insert(const Slice& key, uint32_t hash,
-                             Cache::ObjectPtr value,
-                             const Cache::CacheItemHelper* helper,
-                             size_t charge, LRUHandle** handle,
-                             Cache::Priority priority) {
+LRUHandle* LRUCacheShard::CreateHandle(const Slice& key, uint32_t hash,
+                                       Cache::ObjectPtr value,
+                                       const Cache::CacheItemHelper* helper,
+                                       size_t charge) {
   assert(helper);
+  // value == nullptr is reserved for indicating failure in SecondaryCache
+  assert(!(helper->IsSecondaryCacheCompatible() && value == nullptr));
 
   // Allocate the memory here outside of the mutex.
   // If the cache is full, we'll have to release it.
@@ -695,16 +537,53 @@ Status LRUCacheShard::Insert(const Slice& key, uint32_t hash,
   e->hash = hash;
   e->refs = 0;
   e->next = e->prev = nullptr;
-  e->SetInCache(true);
-  e->SetPriority(priority);
   memcpy(e->key_data, key.data(), key.size());
   e->CalcTotalCharge(charge, metadata_charge_policy_);
 
-  // value == nullptr is reserved for indicating failure for when secondary
-  // cache compatible
-  assert(!(e->IsSecondaryCacheCompatible() && value == nullptr));
+  return e;
+}
 
-  return InsertItem(e, handle, /* free_handle_on_fail */ true);
+Status LRUCacheShard::Insert(const Slice& key, uint32_t hash,
+                             Cache::ObjectPtr value,
+                             const Cache::CacheItemHelper* helper,
+                             size_t charge, LRUHandle** handle,
+                             Cache::Priority priority) {
+  LRUHandle* e = CreateHandle(key, hash, value, helper, charge);
+  e->SetPriority(priority);
+  e->SetInCache(true);
+  return InsertItem(e, handle);
+}
+
+LRUHandle* LRUCacheShard::CreateStandalone(const Slice& key, uint32_t hash,
+                                           Cache::ObjectPtr value,
+                                           const Cache::CacheItemHelper* helper,
+                                           size_t charge,
+                                           bool allow_uncharged) {
+  LRUHandle* e = CreateHandle(key, hash, value, helper, charge);
+  e->SetIsStandalone(true);
+  e->Ref();
+
+  autovector<LRUHandle*> last_reference_list;
+
+  {
+    DMutexLock l(mutex_);
+
+    EvictFromLRU(e->total_charge, &last_reference_list);
+
+    if (strict_capacity_limit_ && (usage_ + e->total_charge) > capacity_) {
+      if (allow_uncharged) {
+        e->total_charge = 0;
+      } else {
+        free(e);
+        e = nullptr;
+      }
+    } else {
+      usage_ += e->total_charge;
+    }
+  }
+
+  NotifyEvicted(last_reference_list);
+  return e;
 }
 
 void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
@@ -733,16 +612,6 @@ void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
   }
 }
 
-bool LRUCacheShard::IsReady(LRUHandle* e) {
-  bool ready = true;
-  if (e->IsPending()) {
-    assert(secondary_cache_);
-    assert(e->sec_handle);
-    ready = e->sec_handle->IsReady();
-  }
-  return ready;
-}
-
 size_t LRUCacheShard::GetUsage() const {
   DMutexLock l(mutex_);
   return usage_;
@@ -782,26 +651,23 @@ LRUCache::LRUCache(size_t capacity, int num_shard_bits,
                    double low_pri_pool_ratio,
                    std::shared_ptr<MemoryAllocator> allocator,
                    bool use_adaptive_mutex,
-                   CacheMetadataChargePolicy metadata_charge_policy,
-                   std::shared_ptr<SecondaryCache> _secondary_cache)
+                   CacheMetadataChargePolicy metadata_charge_policy)
     : ShardedCache(capacity, num_shard_bits, strict_capacity_limit,
-                   std::move(allocator)),
-      secondary_cache_(std::move(_secondary_cache)) {
+                   std::move(allocator)) {
   size_t per_shard = GetPerShardCapacity();
-  SecondaryCache* secondary_cache = secondary_cache_.get();
   MemoryAllocator* alloc = memory_allocator();
+  const EvictionCallback* eviction_callback = &eviction_callback_;
   InitShards([=](LRUCacheShard* cs) {
-    new (cs) LRUCacheShard(
-        per_shard, strict_capacity_limit, high_pri_pool_ratio,
-        low_pri_pool_ratio, use_adaptive_mutex, metadata_charge_policy,
-        /* max_upper_hash_bits */ 32 - num_shard_bits, alloc, secondary_cache);
+    new (cs) LRUCacheShard(per_shard, strict_capacity_limit,
+                           high_pri_pool_ratio, low_pri_pool_ratio,
+                           use_adaptive_mutex, metadata_charge_policy,
+                           /* max_upper_hash_bits */ 32 - num_shard_bits, alloc,
+                           eviction_callback);
   });
 }
 
 Cache::ObjectPtr LRUCache::Value(Handle* handle) {
   auto h = reinterpret_cast<const LRUHandle*>(handle);
-  assert(!h->IsPending() || h->value == nullptr);
-  assert(h->value != &kDummyValue);
   return h->value;
 }
 
@@ -824,42 +690,6 @@ double LRUCache::GetHighPriPoolRatio() {
   return GetShard(0).GetHighPriPoolRatio();
 }
 
-void LRUCache::WaitAll(std::vector<Handle*>& handles) {
-  if (secondary_cache_) {
-    std::vector<SecondaryCacheResultHandle*> sec_handles;
-    sec_handles.reserve(handles.size());
-    for (Handle* handle : handles) {
-      if (!handle) {
-        continue;
-      }
-      LRUHandle* lru_handle = reinterpret_cast<LRUHandle*>(handle);
-      if (!lru_handle->IsPending()) {
-        continue;
-      }
-      sec_handles.emplace_back(lru_handle->sec_handle);
-    }
-    secondary_cache_->WaitAll(sec_handles);
-    for (Handle* handle : handles) {
-      if (!handle) {
-        continue;
-      }
-      LRUHandle* lru_handle = reinterpret_cast<LRUHandle*>(handle);
-      if (!lru_handle->IsPending()) {
-        continue;
-      }
-      GetShard(lru_handle->hash).Promote(lru_handle);
-    }
-  }
-}
-
-void LRUCache::AppendPrintableOptions(std::string& str) const {
-  ShardedCache::AppendPrintableOptions(str);  // options from shard
-  if (secondary_cache_) {
-    str.append("  secondary_cache:\n");
-    str.append(secondary_cache_->GetPrintableOptions());
-  }
-}
-
 }  // namespace lru_cache
 
 std::shared_ptr<Cache> NewLRUCache(
@@ -887,10 +717,14 @@ std::shared_ptr<Cache> NewLRUCache(
   if (num_shard_bits < 0) {
     num_shard_bits = GetDefaultCacheShardBits(capacity);
   }
-  return std::make_shared<LRUCache>(
+  std::shared_ptr<Cache> cache = std::make_shared<LRUCache>(
       capacity, num_shard_bits, strict_capacity_limit, high_pri_pool_ratio,
       low_pri_pool_ratio, std::move(memory_allocator), use_adaptive_mutex,
-      metadata_charge_policy, secondary_cache);
+      metadata_charge_policy);
+  if (secondary_cache) {
+    cache = std::make_shared<CacheWithSecondaryAdapter>(cache, secondary_cache);
+  }
+  return cache;
 }
 
 std::shared_ptr<Cache> NewLRUCache(const LRUCacheOptions& cache_opts) {