@nxtedition/rocksdb 7.1.20 → 7.1.21

package/deps/rocksdb/rocksdb/cache/fast_lru_cache.cc

@@ -173,7 +173,7 @@ inline int LRUHandleTable::FindSlot(const Slice& key,
 LRUCacheShard::LRUCacheShard(size_t capacity, size_t estimated_value_size,
                              bool strict_capacity_limit,
                              CacheMetadataChargePolicy metadata_charge_policy)
-    : CacheShard(metadata_charge_policy),
+    : CacheShardBase(metadata_charge_policy),
       capacity_(capacity),
       strict_capacity_limit_(strict_capacity_limit),
       table_(
@@ -211,27 +211,27 @@ void LRUCacheShard::EraseUnRefEntries() {
 void LRUCacheShard::ApplyToSomeEntries(
     const std::function<void(const Slice& key, void* value, size_t charge,
                              DeleterFn deleter)>& callback,
-    uint32_t average_entries_per_lock, uint32_t* state) {
+    size_t average_entries_per_lock, size_t* state) {
   // The state is essentially going to be the starting hash, which works
   // nicely even if we resize between calls because we use upper-most
   // hash bits for table indexes.
   DMutexLock l(mutex_);
-  uint32_t length_bits = table_.GetLengthBits();
-  uint32_t length = table_.GetTableSize();
+  size_t length_bits = table_.GetLengthBits();
+  size_t length = table_.GetTableSize();
 
   assert(average_entries_per_lock > 0);
   // Assuming we are called with same average_entries_per_lock repeatedly,
   // this simplifies some logic (index_end will not overflow).
   assert(average_entries_per_lock < length || *state == 0);
 
-  uint32_t index_begin = *state >> (32 - length_bits);
-  uint32_t index_end = index_begin + average_entries_per_lock;
+  size_t index_begin = *state >> (sizeof(size_t) * 8u - length_bits);
+  size_t index_end = index_begin + average_entries_per_lock;
   if (index_end >= length) {
     // Going to end
     index_end = length;
-    *state = UINT32_MAX;
+    *state = SIZE_MAX;
   } else {
-    *state = index_end << (32 - length_bits);
+    *state = index_end << (sizeof(size_t) * 8u - length_bits);
   }
 
   table_.ApplyToEntriesRange(
@@ -322,8 +322,7 @@ void LRUCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
 
 Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
                              size_t charge, Cache::DeleterFn deleter,
-                             Cache::Handle** handle,
-                             Cache::Priority /*priority*/) {
+                             LRUHandle** handle, Cache::Priority /*priority*/) {
   if (key.size() != kCacheKeySize) {
     return Status::NotSupported("FastLRUCache only supports key size " +
                                 std::to_string(kCacheKeySize) + "B");
@@ -409,7 +408,7 @@ Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
         if (!h->HasRefs()) {
           h->Ref();
         }
-        *handle = reinterpret_cast<Cache::Handle*>(h);
+        *handle = h;
       }
     }
   }
@@ -422,7 +421,7 @@ Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
   return s;
 }
 
-Cache::Handle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash) {
+LRUHandle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash) {
   LRUHandle* h = nullptr;
   {
     DMutexLock l(mutex_);
@@ -437,23 +436,21 @@ Cache::Handle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash) {
       h->Ref();
     }
   }
-  return reinterpret_cast<Cache::Handle*>(h);
+  return h;
 }
 
-bool LRUCacheShard::Ref(Cache::Handle* h) {
-  LRUHandle* e = reinterpret_cast<LRUHandle*>(h);
+bool LRUCacheShard::Ref(LRUHandle* h) {
   DMutexLock l(mutex_);
   // To create another reference - entry must be already externally referenced.
-  assert(e->HasRefs());
-  e->Ref();
+  assert(h->HasRefs());
+  h->Ref();
   return true;
 }
 
-bool LRUCacheShard::Release(Cache::Handle* handle, bool erase_if_last_ref) {
-  if (handle == nullptr) {
+bool LRUCacheShard::Release(LRUHandle* h, bool erase_if_last_ref) {
+  if (h == nullptr) {
     return false;
   }
-  LRUHandle* h = reinterpret_cast<LRUHandle*>(handle);
   LRUHandle copy;
   bool last_reference = false;
   {
@@ -535,41 +532,18 @@ size_t LRUCacheShard::GetTableAddressCount() const {
   return table_.GetTableSize();
 }
 
-std::string LRUCacheShard::GetPrintableOptions() const { return std::string{}; }
-
 LRUCache::LRUCache(size_t capacity, size_t estimated_value_size,
                    int num_shard_bits, bool strict_capacity_limit,
                    CacheMetadataChargePolicy metadata_charge_policy)
-    : ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
+    : ShardedCache(capacity, num_shard_bits, strict_capacity_limit,
+                   nullptr /*allocator*/) {
   assert(estimated_value_size > 0 ||
          metadata_charge_policy != kDontChargeCacheMetadata);
-  num_shards_ = 1 << num_shard_bits;
-  shards_ = reinterpret_cast<LRUCacheShard*>(
-      port::cacheline_aligned_alloc(sizeof(LRUCacheShard) * num_shards_));
-  size_t per_shard = (capacity + (num_shards_ - 1)) / num_shards_;
-  for (int i = 0; i < num_shards_; i++) {
-    new (&shards_[i])
-        LRUCacheShard(per_shard, estimated_value_size, strict_capacity_limit,
-                      metadata_charge_policy);
-  }
-}
-
-LRUCache::~LRUCache() {
-  if (shards_ != nullptr) {
-    assert(num_shards_ > 0);
-    for (int i = 0; i < num_shards_; i++) {
-      shards_[i].~LRUCacheShard();
-    }
-    port::cacheline_aligned_free(shards_);
-  }
-}
-
-CacheShard* LRUCache::GetShard(uint32_t shard) {
-  return reinterpret_cast<CacheShard*>(&shards_[shard]);
-}
-
-const CacheShard* LRUCache::GetShard(uint32_t shard) const {
-  return reinterpret_cast<CacheShard*>(&shards_[shard]);
+  size_t per_shard = GetPerShardCapacity();
+  InitShards([=](LRUCacheShard* cs) {
+    new (cs) LRUCacheShard(per_shard, estimated_value_size,
+                           strict_capacity_limit, metadata_charge_policy);
+  });
 }
 
 void* LRUCache::Value(Handle* handle) {
@@ -577,12 +551,8 @@ void* LRUCache::Value(Handle* handle) {
 }
 
 size_t LRUCache::GetCharge(Handle* handle) const {
-  CacheMetadataChargePolicy metadata_charge_policy = kDontChargeCacheMetadata;
-  if (num_shards_ > 0) {
-    metadata_charge_policy = shards_[0].metadata_charge_policy_;
-  }
   return reinterpret_cast<const LRUHandle*>(handle)->GetCharge(
-      metadata_charge_policy);
+      GetShard(0).metadata_charge_policy_);
 }
 
 Cache::DeleterFn LRUCache::GetDeleter(Handle* handle) const {
@@ -590,18 +560,6 @@ Cache::DeleterFn LRUCache::GetDeleter(Handle* handle) const {
   return h->deleter;
 }
 
-uint32_t LRUCache::GetHash(Handle* handle) const {
-  return reinterpret_cast<const LRUHandle*>(handle)->hash;
-}
-
-void LRUCache::DisownData() {
-  // Leak data only if that won't generate an ASAN/valgrind warning.
-  if (!kMustFreeHeapAllocations) {
-    shards_ = nullptr;
-    num_shards_ = 0;
-  }
-}
-
 }  // namespace fast_lru_cache
 
 std::shared_ptr<Cache> NewFastLRUCache(

package/deps/rocksdb/rocksdb/cache/fast_lru_cache.h

@@ -141,6 +141,9 @@ struct LRUHandle {
 
   Slice key() const { return Slice(key_data.data(), kCacheKeySize); }
 
+  // For HandleImpl concept
+  uint32_t GetHash() const { return hash; }
+
   // Increase the reference count by 1.
   void Ref() { refs++; }
 
@@ -260,8 +263,8 @@ class LRUHandleTable {
   void Assign(int slot, LRUHandle* h);
 
   template <typename T>
-  void ApplyToEntriesRange(T func, uint32_t index_begin, uint32_t index_end) {
-    for (uint32_t i = index_begin; i < index_end; i++) {
+  void ApplyToEntriesRange(T func, size_t index_begin, size_t index_end) {
+    for (size_t i = index_begin; i < index_end; i++) {
       LRUHandle* h = &array_[i];
       if (h->IsVisible()) {
         func(h);
@@ -316,20 +319,30 @@ class LRUHandleTable {
 };
 
 // A single shard of sharded cache.
-class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
+class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShardBase {
  public:
   LRUCacheShard(size_t capacity, size_t estimated_value_size,
                 bool strict_capacity_limit,
                 CacheMetadataChargePolicy metadata_charge_policy);
-  ~LRUCacheShard() override = default;
+
+  // For CacheShard concept
+  using HandleImpl = LRUHandle;
+
+  // Keep 32-bit hashing for now (FIXME: upgrade to 64-bit)
+  using HashVal = uint32_t;
+  using HashCref = uint32_t;
+  static inline HashVal ComputeHash(const Slice& key) {
+    return Lower32of64(GetSliceNPHash64(key));
+  }
+  static inline uint32_t HashPieceForSharding(HashCref hash) { return hash; }
 
   // Separate from constructor so caller can easily make an array of LRUCache
   // if current usage is more than new capacity, the function will attempt to
   // free the needed space.
-  void SetCapacity(size_t capacity) override;
+  void SetCapacity(size_t capacity);
 
   // Set the flag to reject insertion if cache if full.
-  void SetStrictCapacityLimit(bool strict_capacity_limit) override;
+  void SetStrictCapacityLimit(bool strict_capacity_limit);
 
   // Like Cache methods, but with an extra "hash" parameter.
   // Insert an item into the hash table and, if handle is null, insert into
@@ -337,48 +350,45 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // and free_handle_on_fail is true, the item is deleted and handle is set to
   // nullptr.
   Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge,
-                Cache::DeleterFn deleter, Cache::Handle** handle,
-                Cache::Priority priority) override;
+                Cache::DeleterFn deleter, LRUHandle** handle,
+                Cache::Priority priority);
 
   Status Insert(const Slice& key, uint32_t hash, void* value,
                 const Cache::CacheItemHelper* helper, size_t charge,
-                Cache::Handle** handle, Cache::Priority priority) override {
+                LRUHandle** handle, Cache::Priority priority) {
     return Insert(key, hash, value, charge, helper->del_cb, handle, priority);
   }
 
-  Cache::Handle* Lookup(const Slice& key, uint32_t hash,
-                        const Cache::CacheItemHelper* /*helper*/,
-                        const Cache::CreateCallback& /*create_cb*/,
-                        Cache::Priority /*priority*/, bool /*wait*/,
-                        Statistics* /*stats*/) override {
+  LRUHandle* Lookup(const Slice& key, uint32_t hash,
+                    const Cache::CacheItemHelper* /*helper*/,
+                    const Cache::CreateCallback& /*create_cb*/,
+                    Cache::Priority /*priority*/, bool /*wait*/,
+                    Statistics* /*stats*/) {
     return Lookup(key, hash);
   }
-  Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
+  LRUHandle* Lookup(const Slice& key, uint32_t hash);
 
-  bool Release(Cache::Handle* handle, bool /*useful*/,
-               bool erase_if_last_ref) override {
+  bool Release(LRUHandle* handle, bool /*useful*/, bool erase_if_last_ref) {
     return Release(handle, erase_if_last_ref);
   }
-  bool IsReady(Cache::Handle* /*handle*/) override { return true; }
-  void Wait(Cache::Handle* /*handle*/) override {}
+  bool IsReady(LRUHandle* /*handle*/) { return true; }
+  void Wait(LRUHandle* /*handle*/) {}
 
-  bool Ref(Cache::Handle* handle) override;
-  bool Release(Cache::Handle* handle, bool erase_if_last_ref = false) override;
-  void Erase(const Slice& key, uint32_t hash) override;
+  bool Ref(LRUHandle* handle);
+  bool Release(LRUHandle* handle, bool erase_if_last_ref = false);
+  void Erase(const Slice& key, uint32_t hash);
 
-  size_t GetUsage() const override;
-  size_t GetPinnedUsage() const override;
-  size_t GetOccupancyCount() const override;
-  size_t GetTableAddressCount() const override;
+  size_t GetUsage() const;
+  size_t GetPinnedUsage() const;
+  size_t GetOccupancyCount() const;
+  size_t GetTableAddressCount() const;
 
   void ApplyToSomeEntries(
       const std::function<void(const Slice& key, void* value, size_t charge,
                                DeleterFn deleter)>& callback,
-      uint32_t average_entries_per_lock, uint32_t* state) override;
-
-  void EraseUnRefEntries() override;
+      size_t average_entries_per_lock, size_t* state);
 
-  std::string GetPrintableOptions() const override;
+  void EraseUnRefEntries();
 
  private:
   friend class LRUCache;
@@ -446,25 +456,16 @@ class LRUCache
 #ifdef NDEBUG
     final
 #endif
-    : public ShardedCache {
+    : public ShardedCache<LRUCacheShard> {
  public:
   LRUCache(size_t capacity, size_t estimated_value_size, int num_shard_bits,
            bool strict_capacity_limit,
            CacheMetadataChargePolicy metadata_charge_policy =
                kDontChargeCacheMetadata);
-  ~LRUCache() override;
   const char* Name() const override { return "LRUCache"; }
-  CacheShard* GetShard(uint32_t shard) override;
-  const CacheShard* GetShard(uint32_t shard) const override;
   void* Value(Handle* handle) override;
   size_t GetCharge(Handle* handle) const override;
-  uint32_t GetHash(Handle* handle) const override;
   DeleterFn GetDeleter(Handle* handle) const override;
-  void DisownData() override;
-
- private:
-  LRUCacheShard* shards_ = nullptr;
-  int num_shards_ = 0;
 };
 }  // namespace fast_lru_cache