@nxtedition/rocksdb 7.0.4 → 7.0.7

package/deps/rocksdb/rocksdb/cache/fast_lru_cache.cc

@@ -9,77 +9,191 @@
 
 #include "cache/fast_lru_cache.h"
 
+#include <math.h>
+
 #include <cassert>
 #include <cstdint>
 #include <cstdio>
+#include <functional>
 
 #include "monitoring/perf_context_imp.h"
 #include "monitoring/statistics.h"
 #include "port/lang.h"
-#include "util/mutexlock.h"
-
-#define KEY_LENGTH \
-  16  // TODO(guido) Make use of this symbol in other parts of the source code
-      // (e.g., cache_key.h, cache_test.cc, etc.)
+#include "util/distributed_mutex.h"
+#include "util/hash.h"
+#include "util/random.h"
 
 namespace ROCKSDB_NAMESPACE {
 
 namespace fast_lru_cache {
 
-LRUHandleTable::LRUHandleTable(int hash_bits)
+namespace {
+// Returns x % 2^{bits}.
+inline uint32_t BinaryMod(uint32_t x, uint8_t bits) {
+  assert(bits <= 32);
+  return (x << (32 - bits)) >> (32 - bits);
+}
+}  // anonymous namespace
+
+LRUHandleTable::LRUHandleTable(uint8_t hash_bits)
     : length_bits_(hash_bits),
-      list_(new LRUHandle* [size_t{1} << length_bits_] {}) {}
+      occupancy_(0),
+      array_(new LRUHandle[size_t{1} << length_bits_]) {
+  assert(hash_bits <= 32);
+}
 
 LRUHandleTable::~LRUHandleTable() {
+  // TODO(Guido) If users still hold references to handles,
+  // those will become invalidated. And if we choose not to
+  // delete the data, it will become leaked.
   ApplyToEntriesRange(
       [](LRUHandle* h) {
+        // TODO(Guido) Remove the HasRefs() check?
         if (!h->HasRefs()) {
-          h->Free();
+          h->FreeData();
         }
       },
       0, uint32_t{1} << length_bits_);
 }
 
 LRUHandle* LRUHandleTable::Lookup(const Slice& key, uint32_t hash) {
-  return *FindPointer(key, hash);
-}
-
-inline LRUHandle** LRUHandleTable::Head(uint32_t hash) {
-  return &list_[hash >> (32 - length_bits_)];
-}
-
-LRUHandle* LRUHandleTable::Insert(LRUHandle* h) {
-  LRUHandle** ptr = FindPointer(h->key(), h->hash);
-  LRUHandle* old = *ptr;
-  h->next_hash = (old == nullptr ? nullptr : old->next_hash);
-  *ptr = h;
-  return old;
-}
+  int probe = 0;
+  int slot = FindVisibleElement(key, hash, probe, 0);
+  return (slot == -1) ? nullptr : &array_[slot];
+}
+
+LRUHandle* LRUHandleTable::Insert(LRUHandle* h, LRUHandle** old) {
+  int probe = 0;
+  int slot = FindVisibleElementOrAvailableSlot(h->key(), h->hash, probe,
+                                               1 /*displacement*/);
+  *old = nullptr;
+  if (slot == -1) {
+    return nullptr;
+  }
 
-LRUHandle* LRUHandleTable::Remove(const Slice& key, uint32_t hash) {
-  LRUHandle** ptr = FindPointer(key, hash);
-  LRUHandle* result = *ptr;
-  if (result != nullptr) {
-    *ptr = result->next_hash;
+  if (array_[slot].IsEmpty() || array_[slot].IsTombstone()) {
+    bool empty = array_[slot].IsEmpty();
+    Assign(slot, h);
+    LRUHandle* new_entry = &array_[slot];
+    if (empty) {
+      // This used to be an empty slot.
+      return new_entry;
+    }
+    // It used to be a tombstone, so there may already be a copy of the
+    // key in the table.
+    slot = FindVisibleElement(h->key(), h->hash, probe, 0 /*displacement*/);
+    if (slot == -1) {
+      // No existing copy of the key.
+      return new_entry;
+    }
+    *old = &array_[slot];
+    return new_entry;
+  } else {
+    // There is an existing copy of the key.
+    *old = &array_[slot];
+    // Find an available slot for the new element.
+    array_[slot].displacements++;
+    slot = FindAvailableSlot(h->key(), probe, 1 /*displacement*/);
+    if (slot == -1) {
+      // No available slots. Roll back displacements.
+      probe = 0;
+      slot = FindVisibleElement(h->key(), h->hash, probe, -1);
+      array_[slot].displacements--;
+      FindAvailableSlot(h->key(), probe, -1);
+      return nullptr;
+    }
+    Assign(slot, h);
+    return &array_[slot];
   }
-  return result;
 }
 
-LRUHandle** LRUHandleTable::FindPointer(const Slice& key, uint32_t hash) {
-  LRUHandle** ptr = &list_[hash >> (32 - length_bits_)];
-  while (*ptr != nullptr && ((*ptr)->hash != hash || key != (*ptr)->key())) {
-    ptr = &(*ptr)->next_hash;
+void LRUHandleTable::Remove(LRUHandle* h) {
+  assert(h->next == nullptr &&
+         h->prev == nullptr);  // Already off the LRU list.
+  int probe = 0;
+  FindSlot(
+      h->key(), [&h](LRUHandle* e) { return e == h; }, probe,
+      -1 /*displacement*/);
+  h->SetIsVisible(false);
+  h->SetIsElement(false);
+  occupancy_--;
+}
+
+void LRUHandleTable::Assign(int slot, LRUHandle* h) {
+  LRUHandle* dst = &array_[slot];
+  uint32_t disp = dst->displacements;
+  *dst = *h;
+  dst->displacements = disp;
+  dst->SetIsVisible(true);
+  dst->SetIsElement(true);
+  occupancy_++;
+}
+
+void LRUHandleTable::Exclude(LRUHandle* h) { h->SetIsVisible(false); }
+
+int LRUHandleTable::FindVisibleElement(const Slice& key, uint32_t hash,
+                                       int& probe, int displacement) {
+  return FindSlot(
+      key,
+      [&](LRUHandle* h) { return h->Matches(key, hash) && h->IsVisible(); },
+      probe, displacement);
+}
+
+int LRUHandleTable::FindAvailableSlot(const Slice& key, int& probe,
+                                      int displacement) {
+  return FindSlot(
+      key, [](LRUHandle* h) { return h->IsEmpty() || h->IsTombstone(); }, probe,
+      displacement);
+}
+
+int LRUHandleTable::FindVisibleElementOrAvailableSlot(const Slice& key,
+                                                      uint32_t hash, int& probe,
+                                                      int displacement) {
+  return FindSlot(
+      key,
+      [&](LRUHandle* h) {
+        return h->IsEmpty() || h->IsTombstone() ||
+               (h->Matches(key, hash) && h->IsVisible());
+      },
+      probe, displacement);
+}
+
+inline int LRUHandleTable::FindSlot(const Slice& key,
+                                    std::function<bool(LRUHandle*)> cond,
+                                    int& probe, int displacement) {
+  uint32_t base =
+      BinaryMod(Hash(key.data(), key.size(), kProbingSeed1), length_bits_);
+  uint32_t increment = BinaryMod(
+      (Hash(key.data(), key.size(), kProbingSeed2) << 1) | 1, length_bits_);
+  uint32_t current = BinaryMod(base + probe * increment, length_bits_);
+  while (true) {
+    LRUHandle* h = &array_[current];
+    probe++;
+    if (current == base && probe > 1) {
+      // We looped back.
+      return -1;
+    }
+    if (cond(h)) {
+      return current;
+    }
+    if (h->IsEmpty()) {
+      // We check emptyness after the condition, because
+      // the condition may be emptyness.
+      return -1;
+    }
+    h->displacements += displacement;
+    current = BinaryMod(current + increment, length_bits_);
   }
-  return ptr;
 }
 
 LRUCacheShard::LRUCacheShard(size_t capacity, size_t estimated_value_size,
                              bool strict_capacity_limit,
                              CacheMetadataChargePolicy metadata_charge_policy)
-    : capacity_(0),
+    : capacity_(capacity),
       strict_capacity_limit_(strict_capacity_limit),
       table_(
-          GetHashBits(capacity, estimated_value_size, metadata_charge_policy)),
+          CalcHashBits(capacity, estimated_value_size, metadata_charge_policy) +
+          static_cast<uint8_t>(ceil(log2(1.0 / kLoadFactor)))),
       usage_(0),
       lru_usage_(0) {
   set_metadata_charge_policy(metadata_charge_policy);
@@ -87,29 +201,27 @@ LRUCacheShard::LRUCacheShard(size_t capacity, size_t estimated_value_size,
   lru_.next = &lru_;
   lru_.prev = &lru_;
   lru_low_pri_ = &lru_;
-  SetCapacity(capacity);
 }
 
 void LRUCacheShard::EraseUnRefEntries() {
-  autovector<LRUHandle*> last_reference_list;
+  autovector<LRUHandle> last_reference_list;
   {
-    MutexLock l(&mutex_);
+    DMutexLock l(mutex_);
     while (lru_.next != &lru_) {
       LRUHandle* old = lru_.next;
       // LRU list contains only elements which can be evicted.
-      assert(old->InCache() && !old->HasRefs());
+      assert(old->IsVisible() && !old->HasRefs());
       LRU_Remove(old);
-      table_.Remove(old->key(), old->hash);
-      old->SetInCache(false);
+      table_.Remove(old);
       assert(usage_ >= old->total_charge);
       usage_ -= old->total_charge;
-      last_reference_list.push_back(old);
+      last_reference_list.push_back(*old);
     }
   }
 
   // Free the entries here outside of mutex for performance reasons.
-  for (auto entry : last_reference_list) {
-    entry->Free();
+  for (auto& h : last_reference_list) {
+    h.FreeData();
   }
 }
 
@@ -120,7 +232,7 @@ void LRUCacheShard::ApplyToSomeEntries(
   // The state is essentially going to be the starting hash, which works
   // nicely even if we resize between calls because we use upper-most
   // hash bits for table indexes.
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   uint32_t length_bits = table_.GetLengthBits();
   uint32_t length = uint32_t{1} << length_bits;
 
@@ -148,57 +260,48 @@ void LRUCacheShard::ApplyToSomeEntries(
       index_begin, index_end);
 }
 
-void LRUCacheShard::LRU_Remove(LRUHandle* e) {
-  assert(e->next != nullptr);
-  assert(e->prev != nullptr);
-  e->next->prev = e->prev;
-  e->prev->next = e->next;
-  e->prev = e->next = nullptr;
-  assert(lru_usage_ >= e->total_charge);
-  lru_usage_ -= e->total_charge;
+void LRUCacheShard::LRU_Remove(LRUHandle* h) {
+  assert(h->next != nullptr);
+  assert(h->prev != nullptr);
+  h->next->prev = h->prev;
+  h->prev->next = h->next;
+  h->prev = h->next = nullptr;
+  assert(lru_usage_ >= h->total_charge);
+  lru_usage_ -= h->total_charge;
 }
 
-void LRUCacheShard::LRU_Insert(LRUHandle* e) {
-  assert(e->next == nullptr);
-  assert(e->prev == nullptr);
-  // Inset "e" to head of LRU list.
-  e->next = &lru_;
-  e->prev = lru_.prev;
-  e->prev->next = e;
-  e->next->prev = e;
-  lru_usage_ += e->total_charge;
+void LRUCacheShard::LRU_Insert(LRUHandle* h) {
+  assert(h->next == nullptr);
+  assert(h->prev == nullptr);
+  // Insert h to head of LRU list.
+  h->next = &lru_;
+  h->prev = lru_.prev;
+  h->prev->next = h;
+  h->next->prev = h;
+  lru_usage_ += h->total_charge;
 }
 
 void LRUCacheShard::EvictFromLRU(size_t charge,
-                                 autovector<LRUHandle*>* deleted) {
+                                 autovector<LRUHandle>* deleted) {
   while ((usage_ + charge) > capacity_ && lru_.next != &lru_) {
     LRUHandle* old = lru_.next;
     // LRU list contains only elements which can be evicted.
-    assert(old->InCache() && !old->HasRefs());
+    assert(old->IsVisible() && !old->HasRefs());
     LRU_Remove(old);
-    table_.Remove(old->key(), old->hash);
-    old->SetInCache(false);
+    table_.Remove(old);
     assert(usage_ >= old->total_charge);
     usage_ -= old->total_charge;
-    deleted->push_back(old);
+    deleted->push_back(*old);
   }
 }
 
-int LRUCacheShard::GetHashBits(
+uint8_t LRUCacheShard::CalcHashBits(
     size_t capacity, size_t estimated_value_size,
     CacheMetadataChargePolicy metadata_charge_policy) {
-  LRUHandle* e = reinterpret_cast<LRUHandle*>(
-      new char[sizeof(LRUHandle) - 1 + KEY_LENGTH]);
-  e->key_length = KEY_LENGTH;
-  e->deleter = nullptr;
-  e->refs = 0;
-  e->flags = 0;
-  e->refs = 0;
-
-  e->CalcTotalCharge(estimated_value_size, metadata_charge_policy);
-  size_t num_entries = capacity / e->total_charge;
-  e->Free();
-  int num_hash_bits = 0;
+  LRUHandle h;
+  h.CalcTotalCharge(estimated_value_size, metadata_charge_policy);
+  size_t num_entries = capacity / h.total_charge;
+  uint8_t num_hash_bits = 0;
   while (num_entries >>= 1) {
     ++num_hash_bits;
   }
@@ -206,106 +309,128 @@ int LRUCacheShard::GetHashBits(
 }
 
 void LRUCacheShard::SetCapacity(size_t capacity) {
-  autovector<LRUHandle*> last_reference_list;
+  assert(false);  // Not supported. TODO(Guido) Support it?
+  autovector<LRUHandle> last_reference_list;
   {
-    MutexLock l(&mutex_);
+    DMutexLock l(mutex_);
     capacity_ = capacity;
     EvictFromLRU(0, &last_reference_list);
   }
 
   // Free the entries here outside of mutex for performance reasons.
-  for (auto entry : last_reference_list) {
-    entry->Free();
+  for (auto& h : last_reference_list) {
+    h.FreeData();
   }
 }
 
 void LRUCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   strict_capacity_limit_ = strict_capacity_limit;
 }
 
-Status LRUCacheShard::InsertItem(LRUHandle* e, Cache::Handle** handle,
-                                 bool free_handle_on_fail) {
+Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
+                             size_t charge, Cache::DeleterFn deleter,
+                             Cache::Handle** handle,
+                             Cache::Priority /*priority*/) {
+  if (key.size() != kCacheKeySize) {
+    return Status::NotSupported("FastLRUCache only supports key size " +
+                                std::to_string(kCacheKeySize) + "B");
+  }
+
+  LRUHandle tmp;
+  tmp.value = value;
+  tmp.deleter = deleter;
+  tmp.hash = hash;
+  tmp.CalcTotalCharge(charge, metadata_charge_policy_);
+  for (int i = 0; i < kCacheKeySize; i++) {
+    tmp.key_data[i] = key.data()[i];
+  }
+
   Status s = Status::OK();
-  autovector<LRUHandle*> last_reference_list;
+  autovector<LRUHandle> last_reference_list;
   {
-    MutexLock l(&mutex_);
+    DMutexLock l(mutex_);
 
     // Free the space following strict LRU policy until enough space
     // is freed or the lru list is empty.
-    EvictFromLRU(e->total_charge, &last_reference_list);
-
-    if ((usage_ + e->total_charge) > capacity_ &&
-        (strict_capacity_limit_ || handle == nullptr)) {
-      e->SetInCache(false);
+    EvictFromLRU(tmp.total_charge, &last_reference_list);
+    if ((usage_ + tmp.total_charge > capacity_ &&
+         (strict_capacity_limit_ || handle == nullptr)) ||
+        table_.GetOccupancy() == size_t{1} << table_.GetLengthBits()) {
+      // Originally, when strict_capacity_limit_ == false and handle != nullptr
+      // (i.e., the user wants to immediately get a reference to the new
+      // handle), the insertion would proceed even if the total charge already
+      // exceeds capacity. We can't do this now, because we can't physically
+      // insert a new handle when the table is at maximum occupancy.
+      // TODO(Guido) Some tests (at least two from cache_test, as well as the
+      // stress tests) currently assume the old behavior.
       if (handle == nullptr) {
         // Don't insert the entry but still return ok, as if the entry inserted
         // into cache and get evicted immediately.
-        last_reference_list.push_back(e);
+        last_reference_list.push_back(tmp);
       } else {
-        if (free_handle_on_fail) {
-          delete[] reinterpret_cast<char*>(e);
-          *handle = nullptr;
-        }
         s = Status::Incomplete("Insert failed due to LRU cache being full.");
       }
     } else {
       // Insert into the cache. Note that the cache might get larger than its
       // capacity if not enough space was freed up.
-      LRUHandle* old = table_.Insert(e);
-      usage_ += e->total_charge;
+      LRUHandle* old;
+      LRUHandle* h = table_.Insert(&tmp, &old);
+      assert(h != nullptr);  // Insertions should never fail.
+      usage_ += h->total_charge;
       if (old != nullptr) {
         s = Status::OkOverwritten();
-        assert(old->InCache());
-        old->SetInCache(false);
+        assert(old->IsVisible());
+        table_.Exclude(old);
         if (!old->HasRefs()) {
           // old is on LRU because it's in cache and its reference count is 0.
           LRU_Remove(old);
+          table_.Remove(old);
           assert(usage_ >= old->total_charge);
           usage_ -= old->total_charge;
-          last_reference_list.push_back(old);
+          last_reference_list.push_back(*old);
         }
       }
       if (handle == nullptr) {
-        LRU_Insert(e);
+        LRU_Insert(h);
       } else {
         // If caller already holds a ref, no need to take one here.
-        if (!e->HasRefs()) {
-          e->Ref();
+        if (!h->HasRefs()) {
+          h->Ref();
         }
-        *handle = reinterpret_cast<Cache::Handle*>(e);
+        *handle = reinterpret_cast<Cache::Handle*>(h);
       }
     }
   }
 
   // Free the entries here outside of mutex for performance reasons.
-  for (auto entry : last_reference_list) {
-    entry->Free();
+  for (auto& h : last_reference_list) {
+    h.FreeData();
   }
 
   return s;
 }
 
 Cache::Handle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash) {
-  LRUHandle* e = nullptr;
+  LRUHandle* h = nullptr;
   {
-    MutexLock l(&mutex_);
-    e = table_.Lookup(key, hash);
-    if (e != nullptr) {
-      assert(e->InCache());
-      if (!e->HasRefs()) {
+    DMutexLock l(mutex_);
+    h = table_.Lookup(key, hash);
+    if (h != nullptr) {
+      assert(h->IsVisible());
+      if (!h->HasRefs()) {
         // The entry is in LRU since it's in hash and has no external references
-        LRU_Remove(e);
+        LRU_Remove(h);
       }
-      e->Ref();
+      h->Ref();
     }
   }
-  return reinterpret_cast<Cache::Handle*>(e);
+  return reinterpret_cast<Cache::Handle*>(h);
 }
 
 bool LRUCacheShard::Ref(Cache::Handle* h) {
   LRUHandle* e = reinterpret_cast<LRUHandle*>(h);
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   // To create another reference - entry must be already externally referenced.
   assert(e->HasRefs());
   e->Ref();
@@ -316,101 +441,74 @@ bool LRUCacheShard::Release(Cache::Handle* handle, bool erase_if_last_ref) {
   if (handle == nullptr) {
     return false;
   }
-  LRUHandle* e = reinterpret_cast<LRUHandle*>(handle);
+  LRUHandle* h = reinterpret_cast<LRUHandle*>(handle);
+  LRUHandle copy;
   bool last_reference = false;
   {
-    MutexLock l(&mutex_);
-    last_reference = e->Unref();
-    if (last_reference && e->InCache()) {
+    DMutexLock l(mutex_);
+    last_reference = h->Unref();
+    if (last_reference && h->IsVisible()) {
       // The item is still in cache, and nobody else holds a reference to it.
       if (usage_ > capacity_ || erase_if_last_ref) {
         // The LRU list must be empty since the cache is full.
         assert(lru_.next == &lru_ || erase_if_last_ref);
         // Take this opportunity and remove the item.
-        table_.Remove(e->key(), e->hash);
-        e->SetInCache(false);
+        table_.Remove(h);
       } else {
         // Put the item back on the LRU list, and don't free it.
-        LRU_Insert(e);
+        LRU_Insert(h);
         last_reference = false;
       }
     }
     // If it was the last reference, then decrement the cache usage.
     if (last_reference) {
-      assert(usage_ >= e->total_charge);
-      usage_ -= e->total_charge;
+      assert(usage_ >= h->total_charge);
+      usage_ -= h->total_charge;
+      copy = *h;
     }
   }
 
   // Free the entry here outside of mutex for performance reasons.
   if (last_reference) {
-    e->Free();
+    copy.FreeData();
   }
   return last_reference;
 }
 
-Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
-                             size_t charge, Cache::DeleterFn deleter,
-                             Cache::Handle** handle,
-                             Cache::Priority /*priority*/) {
-  if (key.size() != KEY_LENGTH) {
-    return Status::NotSupported("FastLRUCache only supports key size " +
-                                std::to_string(KEY_LENGTH) + "B");
-  }
-
-  // Allocate the memory here outside of the mutex.
-  // If the cache is full, we'll have to release it.
-  // It shouldn't happen very often though.
-  LRUHandle* e = reinterpret_cast<LRUHandle*>(
-      new char[sizeof(LRUHandle) - 1 + key.size()]);
-
-  e->value = value;
-  e->flags = 0;
-  e->deleter = deleter;
-  e->key_length = key.size();
-  e->hash = hash;
-  e->refs = 0;
-  e->next = e->prev = nullptr;
-  e->SetInCache(true);
-  e->CalcTotalCharge(charge, metadata_charge_policy_);
-  memcpy(e->key_data, key.data(), key.size());
-
-  return InsertItem(e, handle, /* free_handle_on_fail */ true);
-}
-
 void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
-  LRUHandle* e;
+  LRUHandle copy;
   bool last_reference = false;
   {
-    MutexLock l(&mutex_);
-    e = table_.Remove(key, hash);
-    if (e != nullptr) {
-      assert(e->InCache());
-      e->SetInCache(false);
-      if (!e->HasRefs()) {
-        // The entry is in LRU since it's in hash and has no external references
-        LRU_Remove(e);
-        assert(usage_ >= e->total_charge);
-        usage_ -= e->total_charge;
+    DMutexLock l(mutex_);
+    LRUHandle* h = table_.Lookup(key, hash);
+    if (h != nullptr) {
+      table_.Exclude(h);
+      if (!h->HasRefs()) {
+        // The entry is in LRU since it's in cache and has no external
+        // references
+        LRU_Remove(h);
+        table_.Remove(h);
+        assert(usage_ >= h->total_charge);
+        usage_ -= h->total_charge;
         last_reference = true;
+        copy = *h;
       }
     }
   }
-
   // Free the entry here outside of mutex for performance reasons.
   // last_reference will only be true if e != nullptr.
   if (last_reference) {
-    e->Free();
+    copy.FreeData();
   }
 }
 
 size_t LRUCacheShard::GetUsage() const {
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   return usage_;
 }
 
 size_t LRUCacheShard::GetPinnedUsage() const {
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   assert(usage_ >= lru_usage_);
   return usage_ - lru_usage_;
 }