@nxtedition/rocksdb 7.0.4 → 7.0.7

package/deps/rocksdb/rocksdb/cache/fast_lru_cache.h

@@ -8,44 +8,131 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 #pragma once
 
+#include <array>
 #include <memory>
 #include <string>
 
+#include "cache/cache_key.h"
 #include "cache/sharded_cache.h"
 #include "port/lang.h"
 #include "port/malloc.h"
 #include "port/port.h"
 #include "rocksdb/secondary_cache.h"
 #include "util/autovector.h"
+#include "util/distributed_mutex.h"
 
 namespace ROCKSDB_NAMESPACE {
 namespace fast_lru_cache {
 
+// LRU cache implementation using an open-address hash table.
+
+// Every slot in the hash table is an LRUHandle. Because handles can be
+// referenced externally, we can't discard them immediately once they are
+// deleted (via a delete or an LRU eviction) or replaced by a new version
+// (via an insert of the same key). The state of an element is defined by
+// the following two properties:
+// (R) Referenced: An element can be referenced externally (refs > 0), or not.
+//    Importantly, an element can be evicted if and only if it's not
+//    referenced. In particular, when an element becomes referenced, it's
+//    temporarily taken out of the LRU list until all references to it
+//    are dropped.
+// (V) Visible: An element can visible for lookups (IS_VISIBLE set), or not.
+//    Initially, every element is visible. An element that is not visible is
+//    called a ghost.
+// These properties induce 4 different states, with transitions defined as
+// follows:
+// - V --> not V: When a visible element is deleted or replaced by a new
+//    version.
+// - Not V --> V: This cannot happen. A ghost remains in that state until it's
+//    not referenced any more, at which point it's ready to be removed from the
+//    hash table. (A ghost simply waits to transition to the afterlife---it will
+//    never be visible again.)
+// - R --> not R: When all references to an element are dropped.
+// - Not R --> R: When an unreferenced element becomes referenced. This can only
+//    happen if the element is V, since references to an element can only be
+//    created when it's visible.
+
+// Internally, the cache uses an open-addressed hash table to index the handles.
+// We use tombstone counters to keep track of displacements.
+// Because of the tombstones and the two possible visibility states of an
+// element, the table slots can be in 4 different states:
+// 1. Visible element (IS_ELEMENT set and IS_VISIBLE set): The slot contains a
+//    key-value element.
+// 2. Ghost element (IS_ELEMENT set and IS_VISIBLE unset): The slot contains an
+//    element that has been removed, but it's still referenced. It's invisible
+//    to lookups.
+// 3. Tombstone (IS_ELEMENT unset and displacements > 0): The slot contains a
+//    tombstone.
+// 4. Empty (IS_ELEMENT unset and displacements == 0): The slot is unused.
+//    A slot that is an element can further have IS_VISIBLE set or not.
+// When a ghost is removed from the table, it can either transition to being a
+// tombstone or an empty slot, depending on the number of displacements of the
+// slot. In any case, the slot becomes available. When a handle is inserted
+// into that slot, it becomes a visible element again.
+
+constexpr uint8_t kCacheKeySize =
+    static_cast<uint8_t>(sizeof(ROCKSDB_NAMESPACE::CacheKey));
+
+// The load factor p is a real number in (0, 1) such that at all
+// times at most a fraction p of all slots, without counting tombstones,
+// are occupied by elements. This means that the probability that a
+// random probe hits an empty slot is at most p, and thus at most 1/p probes
+// are required on average. We use p = 70%, so between 1 and 2 probes are
+// needed on average.
+// Because the size of the hash table is always rounded up to the next
+// power of 2, p is really an upper bound on the actual load factor---the
+// actual load factor is anywhere between p/2 and p. This is a bit wasteful,
+// but bear in mind that slots only hold metadata, not actual values.
+// Since space cost is dominated by the values (the LSM blocks),
+// overprovisioning the table with metadata only increases the total cache space
+// usage by a tiny fraction.
+constexpr double kLoadFactor = 0.7;
+
+// Arbitrary seeds.
+constexpr uint32_t kProbingSeed1 = 0xbc9f1d34;
+constexpr uint32_t kProbingSeed2 = 0x7a2bb9d5;
+
 // An experimental (under development!) alternative to LRUCache
 
 struct LRUHandle {
   void* value;
   Cache::DeleterFn deleter;
-  LRUHandle* next_hash;
   LRUHandle* next;
   LRUHandle* prev;
   size_t total_charge;  // TODO(opt): Only allow uint32_t?
-  size_t key_length;
   // The hash of key(). Used for fast sharding and comparisons.
   uint32_t hash;
   // The number of external refs to this entry. The cache itself is not counted.
   uint32_t refs;
 
   enum Flags : uint8_t {
-    // Whether this entry is referenced by the hash table.
-    IN_CACHE = (1 << 0),
+    // Whether the handle is visible to Lookups.
+    IS_VISIBLE = (1 << 0),
+    // Whether the slot is in use by an element.
+    IS_ELEMENT = (1 << 1),
   };
   uint8_t flags;
 
-  // Beginning of the key (MUST BE THE LAST FIELD IN THIS STRUCT!)
-  char key_data[1];
+  // The number of elements that hash to this slot or a lower one,
+  // but wind up in a higher slot.
+  uint32_t displacements;
+
+  std::array<char, kCacheKeySize> key_data;
+
+  LRUHandle() {
+    value = nullptr;
+    deleter = nullptr;
+    next = nullptr;
+    prev = nullptr;
+    total_charge = 0;
+    hash = 0;
+    refs = 0;
+    flags = 0;
+    displacements = 0;
+    key_data.fill(0);
+  }
 
-  Slice key() const { return Slice(key_data, key_length); }
+  Slice key() const { return Slice(key_data.data(), kCacheKeySize); }
 
   // Increase the reference count by 1.
   void Ref() { refs++; }
@@ -60,22 +147,31 @@ struct LRUHandle {
   // Return true if there are external refs, false otherwise.
   bool HasRefs() const { return refs > 0; }
 
-  bool InCache() const { return flags & IN_CACHE; }
+  bool IsVisible() const { return flags & IS_VISIBLE; }
 
-  void SetInCache(bool in_cache) {
-    if (in_cache) {
-      flags |= IN_CACHE;
+  void SetIsVisible(bool is_visible) {
+    if (is_visible) {
+      flags |= IS_VISIBLE;
     } else {
-      flags &= ~IN_CACHE;
+      flags &= ~IS_VISIBLE;
     }
   }
 
-  void Free() {
+  bool IsElement() const { return flags & IS_ELEMENT; }
+
+  void SetIsElement(bool is_element) {
+    if (is_element) {
+      flags |= IS_ELEMENT;
+    } else {
+      flags &= ~IS_ELEMENT;
+    }
+  }
+
+  void FreeData() {
     assert(refs == 0);
     if (deleter) {
       (*deleter)(key(), value);
     }
-    delete[] reinterpret_cast<char*>(this);
   }
 
   // Calculate the memory usage by metadata.
@@ -84,13 +180,22 @@ struct LRUHandle {
     if (metadata_charge_policy != kFullChargeCacheMetadata) {
       return 0;
     } else {
-#ifdef ROCKSDB_MALLOC_USABLE_SIZE
-      return malloc_usable_size(
-          const_cast<void*>(static_cast<const void*>(this)));
-#else
-      // This is the size that is used when a new handle is created.
-      return sizeof(LRUHandle) - 1 + key_length;
-#endif
+      // #ifdef ROCKSDB_MALLOC_USABLE_SIZE
+      //       return malloc_usable_size(
+      //           const_cast<void*>(static_cast<const void*>(this)));
+      // #else
+      // TODO(Guido) malloc_usable_size only works when we call it on
+      // a pointer allocated with malloc. Because our handles are all
+      // allocated in a single shot as an array, the user can't call
+      // CalcMetaCharge (or CalcTotalCharge or GetCharge) on a handle
+      // pointer returned by the cache. Moreover, malloc_usable_size
+      // expects a heap-allocated handle, but sometimes in our code we
+      // wish to pass a stack-allocated handle (this is only a performance
+      // concern).
+      // What is the right way to compute metadata charges with pre-allocated
+      // handles?
+      return sizeof(LRUHandle);
+      // #endif
     }
   }
 
@@ -105,8 +210,23 @@ struct LRUHandle {
     assert(total_charge >= meta_charge);
     return total_charge - meta_charge;
   }
+
+  inline bool IsEmpty() {
+    return !this->IsElement() && this->displacements == 0;
+  }
+
+  inline bool IsTombstone() {
+    return !this->IsElement() && this->displacements > 0;
+  }
+
+  inline bool Matches(const Slice& some_key, uint32_t some_hash) {
+    return this->IsElement() && this->hash == some_hash &&
+           this->key() == some_key;
+  }
 };
 
+// TODO(Guido) Update the following comment.
+
 // We provide our own simple hash table since it removes a whole bunch
 // of porting hacks and is also faster than some of the built-in hash
 // table implementations in some of the compiler/runtime combinations
@@ -114,45 +234,72 @@ struct LRUHandle {
 // 4.4.3's builtin hashtable.
 class LRUHandleTable {
  public:
-  explicit LRUHandleTable(int hash_bits);
+  explicit LRUHandleTable(uint8_t hash_bits);
   ~LRUHandleTable();
 
+  // Returns a pointer to a visible element matching the key/hash, or
+  // nullptr if not present.
   LRUHandle* Lookup(const Slice& key, uint32_t hash);
-  LRUHandle* Insert(LRUHandle* h);
-  LRUHandle* Remove(const Slice& key, uint32_t hash);
+
+  // Inserts a copy of h into the hash table.
+  // Returns a pointer to the inserted handle, or nullptr if no slot
+  // available was found. If an existing visible element matching the
+  // key/hash is already present in the hash table, the argument old
+  // is set to pointe to it; otherwise, it's set to nullptr.
+  LRUHandle* Insert(LRUHandle* h, LRUHandle** old);
+
+  // Removes h from the hash table. The handle must already be off
+  // the LRU list.
+  void Remove(LRUHandle* h);
+
+  // Turns a visible element h into a ghost (i.e., not visible).
+  void Exclude(LRUHandle* h);
+
+  // Assigns a copy of h to the given slot.
+  void Assign(int slot, LRUHandle* h);
 
   template <typename T>
   void ApplyToEntriesRange(T func, uint32_t index_begin, uint32_t index_end) {
     for (uint32_t i = index_begin; i < index_end; i++) {
-      LRUHandle* h = list_[i];
-      while (h != nullptr) {
-        auto n = h->next_hash;
-        assert(h->InCache());
+      LRUHandle* h = &array_[i];
+      if (h->IsVisible()) {
         func(h);
-        h = n;
       }
     }
   }
 
-  int GetLengthBits() const { return length_bits_; }
+  uint8_t GetLengthBits() const { return length_bits_; }
 
-  // Return the address of the head of the chain in the bucket given
-  // by the hash.
-  inline LRUHandle** Head(uint32_t hash);
+  uint32_t GetOccupancy() const { return occupancy_; }
 
  private:
-  // Return a pointer to slot that points to a cache entry that
-  // matches key/hash.  If there is no such cache entry, return a
-  // pointer to the trailing slot in the corresponding linked list.
-  LRUHandle** FindPointer(const Slice& key, uint32_t hash);
-
-  // Number of hash bits (upper because lower bits used for sharding)
-  // used for table index. Length == 1 << length_bits_
-  int length_bits_;
-
-  // The table consists of an array of buckets where each bucket is
-  // a linked list of cache entries that hash into the bucket.
-  std::unique_ptr<LRUHandle*[]> list_;
+  int FindVisibleElement(const Slice& key, uint32_t hash, int& probe,
+                         int displacement);
+
+  int FindAvailableSlot(const Slice& key, int& probe, int displacement);
+
+  int FindVisibleElementOrAvailableSlot(const Slice& key, uint32_t hash,
+                                        int& probe, int displacement);
+
+  // Returns the index of the first slot probed (hashing with
+  // the given key) with a handle e such that cond(e) is true.
+  // Otherwise, if no match is found, returns -1.
+  // For every handle e probed except the final slot, updates
+  // e->displacements += displacement.
+  // The argument probe is modified such that consecutive calls
+  // to FindSlot continue probing right after where the previous
+  // call left.
+  int FindSlot(const Slice& key, std::function<bool(LRUHandle*)> cond,
+               int& probe, int displacement);
+
+  // Number of hash bits used for table index.
+  // The size of the table is 1 << length_bits_.
+  uint8_t length_bits_;
+
+  // Number of elements in the table.
+  uint32_t occupancy_;
+
+  std::unique_ptr<LRUHandle[]> array_;
 };
 
 // A single shard of sharded cache.
@@ -172,6 +319,10 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   void SetStrictCapacityLimit(bool strict_capacity_limit) override;
 
   // Like Cache methods, but with an extra "hash" parameter.
+  // Insert an item into the hash table and, if handle is null, insert into
+  // the LRU list. Older items are evicted as necessary. If the cache is full
+  // and free_handle_on_fail is true, the item is deleted and handle is set to
+  // nullptr.
   Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge,
                 Cache::DeleterFn deleter, Cache::Handle** handle,
                 Cache::Priority priority) override;
@@ -216,13 +367,6 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
 
  private:
   friend class LRUCache;
-  // Insert an item into the hash table and, if handle is null, insert into
-  // the LRU list. Older items are evicted as necessary. If the cache is full
-  // and free_handle_on_fail is true, the item is deleted and handle is set to
-  // nullptr.
-  Status InsertItem(LRUHandle* item, Cache::Handle** handle,
-                    bool free_handle_on_fail);
-
   void LRU_Remove(LRUHandle* e);
   void LRU_Insert(LRUHandle* e);
 
@@ -230,12 +374,12 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // to hold (usage_ + charge) is freed or the lru list is empty
   // This function is not thread safe - it needs to be executed while
   // holding the mutex_.
-  void EvictFromLRU(size_t charge, autovector<LRUHandle*>* deleted);
+  void EvictFromLRU(size_t charge, autovector<LRUHandle>* deleted);
 
-  // Returns the number of bits used to hash an element in the per-shard
+  // Returns the number of bits used to hash an element in the hash
   // table.
-  static int GetHashBits(size_t capacity, size_t estimated_value_size,
-                         CacheMetadataChargePolicy metadata_charge_policy);
+  static uint8_t CalcHashBits(size_t capacity, size_t estimated_value_size,
+                              CacheMetadataChargePolicy metadata_charge_policy);
 
   // Initialized before use.
   size_t capacity_;
@@ -273,7 +417,7 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // mutex_ protects the following state.
   // We don't count mutex_ as the cache's internal state so semantically we
   // don't mind mutex_ invoking the non-const actions.
-  mutable port::Mutex mutex_;
+  mutable DMutex mutex_;
 };
 
 class LRUCache

package/deps/rocksdb/rocksdb/cache/lru_cache.cc

@@ -16,7 +16,7 @@
 #include "monitoring/perf_context_imp.h"
 #include "monitoring/statistics.h"
 #include "port/lang.h"
-#include "util/mutexlock.h"
+#include "util/distributed_mutex.h"
 
 namespace ROCKSDB_NAMESPACE {
 namespace lru_cache {
@@ -135,7 +135,7 @@ LRUCacheShard::LRUCacheShard(
 void LRUCacheShard::EraseUnRefEntries() {
   autovector<LRUHandle*> last_reference_list;
   {
-    MutexLock l(&mutex_);
+    DMutexLock l(mutex_);
     while (lru_.next != &lru_) {
       LRUHandle* old = lru_.next;
       // LRU list contains only elements which can be evicted.
@@ -161,7 +161,7 @@ void LRUCacheShard::ApplyToSomeEntries(
   // The state is essentially going to be the starting hash, which works
   // nicely even if we resize between calls because we use upper-most
   // hash bits for table indexes.
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   uint32_t length_bits = table_.GetLengthBits();
   uint32_t length = uint32_t{1} << length_bits;
 
@@ -193,13 +193,13 @@ void LRUCacheShard::ApplyToSomeEntries(
 }
 
 void LRUCacheShard::TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri) {
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   *lru = &lru_;
   *lru_low_pri = lru_low_pri_;
 }
 
 size_t LRUCacheShard::TEST_GetLRUSize() {
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   LRUHandle* lru_handle = lru_.next;
   size_t lru_size = 0;
   while (lru_handle != &lru_) {
@@ -210,7 +210,7 @@ size_t LRUCacheShard::TEST_GetLRUSize() {
 }
 
 double LRUCacheShard::GetHighPriPoolRatio() {
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   return high_pri_pool_ratio_;
 }
 
@@ -285,7 +285,7 @@ void LRUCacheShard::EvictFromLRU(size_t charge,
 void LRUCacheShard::SetCapacity(size_t capacity) {
   autovector<LRUHandle*> last_reference_list;
   {
-    MutexLock l(&mutex_);
+    DMutexLock l(mutex_);
     capacity_ = capacity;
     high_pri_pool_capacity_ = capacity_ * high_pri_pool_ratio_;
     EvictFromLRU(0, &last_reference_list);
@@ -304,7 +304,7 @@ void LRUCacheShard::SetCapacity(size_t capacity) {
 }
 
 void LRUCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   strict_capacity_limit_ = strict_capacity_limit;
 }
 
@@ -314,7 +314,7 @@ Status LRUCacheShard::InsertItem(LRUHandle* e, Cache::Handle** handle,
   autovector<LRUHandle*> last_reference_list;
 
   {
-    MutexLock l(&mutex_);
+    DMutexLock l(mutex_);
 
     // Free the space following strict LRU policy until enough space
     // is freed or the lru list is empty.
@@ -402,7 +402,7 @@ void LRUCacheShard::Promote(LRUHandle* e) {
   } else {
     // Since the secondary cache lookup failed, mark the item as not in cache
     // Don't charge the cache as its only metadata that'll shortly be released
-    MutexLock l(&mutex_);
+    DMutexLock l(mutex_);
     // TODO
     e->CalcTotalCharge(0, metadata_charge_policy_);
     e->SetInCache(false);
@@ -416,7 +416,7 @@ Cache::Handle* LRUCacheShard::Lookup(
     bool wait, Statistics* stats) {
   LRUHandle* e = nullptr;
   {
-    MutexLock l(&mutex_);
+    DMutexLock l(mutex_);
     e = table_.Lookup(key, hash);
     if (e != nullptr) {
       assert(e->InCache());
@@ -489,7 +489,7 @@ Cache::Handle* LRUCacheShard::Lookup(
 
 bool LRUCacheShard::Ref(Cache::Handle* h) {
   LRUHandle* e = reinterpret_cast<LRUHandle*>(h);
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   // To create another reference - entry must be already externally referenced.
   assert(e->HasRefs());
   e->Ref();
@@ -497,7 +497,7 @@ bool LRUCacheShard::Ref(Cache::Handle* h) {
 }
 
 void LRUCacheShard::SetHighPriorityPoolRatio(double high_pri_pool_ratio) {
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   high_pri_pool_ratio_ = high_pri_pool_ratio;
   high_pri_pool_capacity_ = capacity_ * high_pri_pool_ratio_;
   MaintainPoolSize();
@@ -510,7 +510,7 @@ bool LRUCacheShard::Release(Cache::Handle* handle, bool erase_if_last_ref) {
   LRUHandle* e = reinterpret_cast<LRUHandle*>(handle);
   bool last_reference = false;
   {
-    MutexLock l(&mutex_);
+    DMutexLock l(mutex_);
     last_reference = e->Unref();
     if (last_reference && e->InCache()) {
       // The item is still in cache, and nobody else holds a reference to it.
@@ -582,7 +582,7 @@ void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
   LRUHandle* e;
   bool last_reference = false;
   {
-    MutexLock l(&mutex_);
+    DMutexLock l(mutex_);
     e = table_.Remove(key, hash);
     if (e != nullptr) {
       assert(e->InCache());
@@ -606,7 +606,7 @@ void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
 
 bool LRUCacheShard::IsReady(Cache::Handle* handle) {
   LRUHandle* e = reinterpret_cast<LRUHandle*>(handle);
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   bool ready = true;
   if (e->IsPending()) {
     assert(secondary_cache_);
@@ -617,12 +617,12 @@ bool LRUCacheShard::IsReady(Cache::Handle* handle) {
 }
 
 size_t LRUCacheShard::GetUsage() const {
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   return usage_;
 }
 
 size_t LRUCacheShard::GetPinnedUsage() const {
-  MutexLock l(&mutex_);
+  DMutexLock l(mutex_);
   assert(usage_ >= lru_usage_);
   return usage_ - lru_usage_;
 }
@@ -631,7 +631,7 @@ std::string LRUCacheShard::GetPrintableOptions() const {
   const int kBufferSize = 200;
   char buffer[kBufferSize];
   {
-    MutexLock l(&mutex_);
+    DMutexLock l(mutex_);
     snprintf(buffer, kBufferSize, "    high_pri_pool_ratio: %.3lf\n",
              high_pri_pool_ratio_);
   }

package/deps/rocksdb/rocksdb/cache/lru_cache.h

@@ -17,6 +17,7 @@
 #include "port/port.h"
 #include "rocksdb/secondary_cache.h"
 #include "util/autovector.h"
+#include "util/distributed_mutex.h"
 
 namespace ROCKSDB_NAMESPACE {
 namespace lru_cache {
@@ -453,7 +454,7 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // mutex_ protects the following state.
   // We don't count mutex_ as the cache's internal state so semantically we
   // don't mind mutex_ invoking the non-const actions.
-  mutable port::Mutex mutex_;
+  mutable DMutex mutex_;
 
   std::shared_ptr<SecondaryCache> secondary_cache_;
 };