@nxtedition/rocksdb 7.1.4 → 7.1.7

package/deps/rocksdb/rocksdb/cache/clock_cache.cc

@@ -69,10 +69,10 @@ ClockHandle* ClockHandleTable::Lookup(const Slice& key, uint32_t hash) {
     // updates where it would be possible to combine into one CAS (more metadata
     // under one atomic field) or maybe two atomic updates (one arithmetic, one
     // bitwise). Something to think about optimizing.
-    e->InternalToExternalRef();
     e->SetHit();
     // The handle is now referenced, so we take it out of clock.
     ClockOff(e);
+    e->InternalToExternalRef();
   }
 
   return e;
@@ -312,17 +312,20 @@ void ClockHandleTable::ClockRun(size_t charge) {
   // hot element, it will be hard to get an exclusive ref.
   // Do we need a mechanism to prevent an element from sitting
   // for a long time in cache waiting to be evicted?
-  assert(charge <= capacity_);
   autovector<ClockHandle> deleted;
   uint32_t max_iterations =
-      1 + static_cast<uint32_t>(GetTableSize() * kLoadFactor);
+      ClockHandle::ClockPriority::HIGH *
+      (1 +
+       static_cast<uint32_t>(
+           GetTableSize() *
+           kLoadFactor));  // It may take up to HIGH passes to evict an element.
   size_t usage_local = usage_;
-  while (usage_local + charge > capacity_ && max_iterations--) {
+  size_t capacity_local = capacity_;
+  while (usage_local + charge > capacity_local && max_iterations--) {
     uint32_t steps = 1 + static_cast<uint32_t>(1 / kLoadFactor);
     uint32_t clock_pointer_local = (clock_pointer_ += steps) - steps;
     for (uint32_t i = 0; i < steps; i++) {
       ClockHandle* h = &array_[ModTableSize(clock_pointer_local + i)];
-
       if (h->TryExclusiveRef()) {
         if (h->WillBeDeleted()) {
           Remove(h, &deleted);
@@ -335,7 +338,6 @@ void ClockHandleTable::ClockRun(size_t charge) {
             // exclusive ref, we know we are in the latter case. This can only
             // happen when the last external reference to an element was
             // released, and the element was not immediately removed.
-
             ClockOn(h);
           }
           ClockHandle::ClockPriority priority = h->GetClockPriority();
@@ -358,6 +360,7 @@ ClockCacheShard::ClockCacheShard(
     size_t capacity, size_t estimated_value_size, bool strict_capacity_limit,
     CacheMetadataChargePolicy metadata_charge_policy)
     : strict_capacity_limit_(strict_capacity_limit),
+      detached_usage_(0),
       table_(capacity, CalcHashBits(capacity, estimated_value_size,
                                     metadata_charge_policy)) {
   set_metadata_charge_policy(metadata_charge_policy);
@@ -401,15 +404,24 @@ void ClockCacheShard::ApplyToSomeEntries(
     *state = index_end << (32 - length_bits);
   }
 
-  table_.ApplyToEntriesRange(
+  table_.ConstApplyToEntriesRange(
       [callback,
-       metadata_charge_policy = metadata_charge_policy_](ClockHandle* h) {
+       metadata_charge_policy = metadata_charge_policy_](const ClockHandle* h) {
         callback(h->key(), h->value, h->GetCharge(metadata_charge_policy),
                  h->deleter);
       },
       index_begin, index_end, false);
 }
 
+ClockHandle* ClockCacheShard::DetachedInsert(ClockHandle* h) {
+  ClockHandle* e = new ClockHandle();
+  *e = *h;
+  e->SetDetached();
+  e->TryExternalRef();
+  detached_usage_ += h->total_charge;
+  return e;
+}
+
 size_t ClockCacheShard::CalcEstimatedHandleCharge(
     size_t estimated_value_size,
     CacheMetadataChargePolicy metadata_charge_policy) {
@@ -430,12 +442,16 @@ int ClockCacheShard::CalcHashBits(
   return FloorLog2((num_entries << 1) - 1);
 }
 
-void ClockCacheShard::SetCapacity(size_t /*capacity*/) {
-  assert(false);  // Not supported.
+void ClockCacheShard::SetCapacity(size_t capacity) {
+  if (capacity > table_.GetCapacity()) {
+    assert(false);  // Not supported.
+  }
+  table_.SetCapacity(capacity);
+  table_.ClockRun(detached_usage_);
 }
 
-void ClockCacheShard::SetStrictCapacityLimit(bool /*strict_capacity_limit*/) {
-  assert(false);  // Not supported.
+void ClockCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
+  strict_capacity_limit_ = strict_capacity_limit;
 }
 
 Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
@@ -459,27 +475,32 @@ Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
 
   Status s = Status::OK();
 
+  // Use a local copy to minimize cache synchronization.
+  size_t detached_usage = detached_usage_;
+
   // Free space with the clock policy until enough space is freed or there are
   // no evictable elements.
-  table_.ClockRun(tmp.total_charge);
+  table_.ClockRun(tmp.total_charge + detached_usage);
 
-  // occupancy_ and usage_ are contended members across concurrent updates
-  // on the same shard, so we use a single copy to reduce cache synchronization.
+  // Use local copies to minimize cache synchronization
+  // (occupancy_ and usage_ are read and written by all insertions).
   uint32_t occupancy_local = table_.GetOccupancy();
-  size_t usage_local = table_.GetUsage();
-  assert(occupancy_local <= table_.GetOccupancyLimit());
-
-  autovector<ClockHandle> deleted;
-
-  if ((usage_local + tmp.total_charge > table_.GetCapacity() &&
-       (strict_capacity_limit_ || handle == nullptr)) ||
-      occupancy_local > table_.GetOccupancyLimit()) {
+  size_t total_usage = table_.GetUsage() + detached_usage;
+
+  // TODO: Currently we support strict_capacity_limit == false as long as the
+  // number of pinned elements is below table_.GetOccupancyLimit(). We can
+  // always support it as follows: whenever we exceed this limit, we dynamically
+  // allocate a handle and return it (when the user provides a handle pointer,
+  // of course). Then, Release checks whether the handle was dynamically
+  // allocated, or is stored in the table.
+  if (total_usage + tmp.total_charge > table_.GetCapacity() &&
+      (strict_capacity_limit_ || handle == nullptr)) {
     if (handle == nullptr) {
       // Don't insert the entry but still return ok, as if the entry inserted
       // into cache and get evicted immediately.
-      deleted.push_back(tmp);
+      tmp.FreeData();
     } else {
-      if (occupancy_local > table_.GetOccupancyLimit()) {
+      if (occupancy_local + 1 > table_.GetOccupancyLimit()) {
         // TODO: Consider using a distinct status for this case, but usually
         // it will be handled the same way as reaching charge capacity limit
         s = Status::MemoryLimit(
@@ -491,22 +512,34 @@ Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
       }
     }
   } else {
-    // Insert into the cache. Note that the cache might get larger than its
-    // capacity if not enough space was freed up.
-    ClockHandle* h = table_.Insert(&tmp, &deleted, handle != nullptr);
-    assert(h != nullptr);  // The occupancy is way below the table size, so this
-                           // insertion should never fail.
+    ClockHandle* h = nullptr;
+    if (handle != nullptr && occupancy_local + 1 > table_.GetOccupancyLimit()) {
+      // Even if the user wishes to overload the cache, we can't insert into
+      // the hash table. Instead, we dynamically allocate a new handle.
+      h = DetachedInsert(&tmp);
+      // TODO: Return special status?
+    } else {
+      // Insert into the cache. Note that the cache might get larger than its
+      // capacity if not enough space was freed up.
+      autovector<ClockHandle> deleted;
+      h = table_.Insert(&tmp, &deleted, handle != nullptr);
+      if (h == nullptr && handle != nullptr) {
+        // The table is full. This can happen when many threads simultaneously
+        // attempt an insert, and the table is operating close to full capacity.
+        h = DetachedInsert(&tmp);
+      }
+      // Notice that if handle == nullptr, we don't insert the entry but still
+      // return ok.
+      if (deleted.size() > 0) {
+        s = Status::OkOverwritten();
+      }
+      table_.Free(&deleted);
+    }
     if (handle != nullptr) {
       *handle = reinterpret_cast<Cache::Handle*>(h);
     }
-
-    if (deleted.size() > 0) {
-      s = Status::OkOverwritten();
-    }
   }
 
-  table_.Free(&deleted);
-
   return s;
 }
 
@@ -516,7 +549,7 @@ Cache::Handle* ClockCacheShard::Lookup(const Slice& key, uint32_t hash) {
 
 bool ClockCacheShard::Ref(Cache::Handle* h) {
   ClockHandle* e = reinterpret_cast<ClockHandle*>(h);
-  assert(e->HasExternalRefs());
+  assert(e->ExternalRefs() > 0);
   return e->TryExternalRef();
 }
 
@@ -530,6 +563,20 @@ bool ClockCacheShard::Release(Cache::Handle* handle, bool erase_if_last_ref) {
   }
 
   ClockHandle* h = reinterpret_cast<ClockHandle*>(handle);
+
+  if (UNLIKELY(h->IsDetached())) {
+    h->ReleaseExternalRef();
+    if (h->TryExclusiveRef()) {
+      // Only the last reference will succeed.
+      // Don't bother releasing the exclusive ref.
+      h->FreeData();
+      detached_usage_ -= h->total_charge;
+      delete h;
+      return true;
+    }
+    return false;
+  }
+
   uint32_t refs = h->refs;
   bool last_reference = ((refs & ClockHandle::EXTERNAL_REFS) == 1);
   bool will_be_deleted = refs & ClockHandle::WILL_BE_DELETED;
@@ -569,14 +616,15 @@ size_t ClockCacheShard::GetPinnedUsage() const {
   size_t clock_usage = 0;
 
   table_.ConstApplyToEntriesRange(
-      [&clock_usage](ClockHandle* h) {
-        if (h->HasExternalRefs()) {
+      [&clock_usage](const ClockHandle* h) {
+        if (h->ExternalRefs() > 1) {
+          // We check > 1 because we are holding an external ref.
           clock_usage += h->total_charge;
         }
       },
       0, table_.GetTableSize(), true);
 
-  return clock_usage;
+  return clock_usage + detached_usage_;
 }
 
 ClockCache::ClockCache(size_t capacity, size_t estimated_value_size,
@@ -649,8 +697,10 @@ void ClockCache::DisownData() {
 std::shared_ptr<Cache> NewClockCache(
     size_t capacity, int num_shard_bits, bool strict_capacity_limit,
     CacheMetadataChargePolicy metadata_charge_policy) {
-  return NewLRUCache(capacity, num_shard_bits, strict_capacity_limit, 0.5,
-                     nullptr, kDefaultToAdaptiveMutex, metadata_charge_policy);
+  return NewLRUCache(capacity, num_shard_bits, strict_capacity_limit,
+                     /* high_pri_pool_ratio */ 0.5, nullptr,
+                     kDefaultToAdaptiveMutex, metadata_charge_policy,
+                     /* low_pri_pool_ratio */ 0.0);
 }
 
 std::shared_ptr<Cache> ExperimentalNewClockCache(

package/deps/rocksdb/rocksdb/cache/clock_cache.h

@@ -9,6 +9,8 @@
 
 #pragma once
 
+#include <sys/types.h>
+
 #include <array>
 #include <atomic>
 #include <cstdint>
@@ -28,6 +30,9 @@ namespace ROCKSDB_NAMESPACE {
 
 namespace clock_cache {
 
+// Forward declaration of friend class.
+class ClockCacheTest;
+
 // An experimental alternative to LRUCache, using a lock-free, open-addressed
 // hash table and clock eviction.
 
@@ -63,10 +68,10 @@ namespace clock_cache {
 //    can't be immediately deleted. In these cases, the flag will be later read
 //    and acted upon by the eviction algorithm. Importantly, WILL_BE_DELETED is
 //    used not only to defer deletions, but also as a barrier for external
-//    references: once WILL_BE_DELETED is set, lookups (which are the means to
-//    acquire new external references) will ignore the handle. For this reason,
-//    when WILL_BE_DELETED is set, we say the handle is invisible (and
-//    otherwise, that it's visible).
+//    references: once WILL_BE_DELETED is set, lookups (which are the most
+//    common way to acquire new external references) will ignore the handle.
+//    For this reason, when WILL_BE_DELETED is set, we say the handle is
+//    invisible (and, otherwise, that it's visible).
 //
 //
 // 3. HASHING AND COLLISION RESOLUTION
@@ -192,10 +197,10 @@ struct ClockHandle {
   size_t total_charge;
   std::array<char, kCacheKeySize> key_data;
 
-  static constexpr uint8_t kIsElementOffset = 1;
-  static constexpr uint8_t kClockPriorityOffset = 2;
-  static constexpr uint8_t kIsHitOffset = 4;
-  static constexpr uint8_t kCachePriorityOffset = 5;
+  static constexpr uint8_t kIsElementOffset = 0;
+  static constexpr uint8_t kClockPriorityOffset = 1;
+  static constexpr uint8_t kIsHitOffset = 3;
+  static constexpr uint8_t kCachePriorityOffset = 4;
 
   enum Flags : uint8_t {
     // Whether the slot is in use by an element.
@@ -252,9 +257,8 @@ struct ClockHandle {
     // Whether a thread has an exclusive reference to the slot.
     EXCLUSIVE_REF = uint32_t{1} << kExclusiveRefOffset,  // Bit 30
     // Whether the handle will be deleted soon. When this bit is set, new
-    // internal
-    // or external references to this handle stop being accepted.
-    // There is an exception: external references can be created from
+    // internal references to this handle stop being accepted.
+    // External references may still be granted---they can be created from
     // existing external references, or converting from existing internal
     // references.
     WILL_BE_DELETED = uint32_t{1} << kWillBeDeletedOffset  // Bit 31
@@ -274,6 +278,9 @@ struct ClockHandle {
 
   std::atomic<uint32_t> refs;
 
+  // True iff the handle is allocated separately from hash table.
+  bool detached;
+
   ClockHandle()
       : value(nullptr),
         deleter(nullptr),
@@ -281,7 +288,8 @@ struct ClockHandle {
         total_charge(0),
         flags(0),
         displacements(0),
-        refs(0) {
+        refs(0),
+        detached(false) {
     SetWillBeDeleted(false);
     SetIsElement(false);
     SetClockPriority(ClockPriority::NONE);
@@ -300,6 +308,7 @@ struct ClockHandle {
     value = other.value;
     deleter = other.deleter;
     key_data = other.key_data;
+    hash = other.hash;
     total_charge = other.total_charge;
   }
 
@@ -350,13 +359,13 @@ struct ClockHandle {
 
   // flags functions.
 
-  bool IsElement() const { return flags & IS_ELEMENT; }
+  bool IsElement() const { return flags & Flags::IS_ELEMENT; }
 
   void SetIsElement(bool is_element) {
     if (is_element) {
-      flags |= IS_ELEMENT;
+      flags |= Flags::IS_ELEMENT;
     } else {
-      flags &= static_cast<uint8_t>(~IS_ELEMENT);
+      flags &= static_cast<uint8_t>(~Flags::IS_ELEMENT);
     }
   }
 
@@ -400,6 +409,10 @@ struct ClockHandle {
     flags |= new_priority;
   }
 
+  bool IsDetached() { return detached; }
+
+  void SetDetached() { detached = true; }
+
   inline bool IsEmpty() const {
     return !this->IsElement() && this->displacements == 0;
   }
@@ -424,7 +437,9 @@ struct ClockHandle {
     }
   }
 
-  bool HasExternalRefs() const { return (refs & EXTERNAL_REFS) > 0; }
+  uint32_t ExternalRefs() const {
+    return (refs & EXTERNAL_REFS) >> kExternalRefsOffset;
+  }
 
   // Tries to take an internal ref. Returns true iff it succeeds.
   inline bool TryInternalRef() {
@@ -437,7 +452,7 @@ struct ClockHandle {
 
   // Tries to take an external ref. Returns true iff it succeeds.
   inline bool TryExternalRef() {
-    if (!((refs += kOneExternalRef) & (EXCLUSIVE_REF | WILL_BE_DELETED))) {
+    if (!((refs += kOneExternalRef) & EXCLUSIVE_REF)) {
       return true;
     }
     refs -= kOneExternalRef;
@@ -529,8 +544,8 @@ class ClockHandleTable {
   // Makes h non-evictable.
   void ClockOff(ClockHandle* h);
 
-  // Runs the clock eviction algorithm until there is enough space to
-  // insert an element with the given charge.
+  // Runs the clock eviction algorithm until usage_ + charge is at most
+  // capacity_.
   void ClockRun(size_t charge);
 
   // Remove h from the hash table. Requires an exclusive ref to h.
@@ -548,8 +563,6 @@ class ClockHandleTable {
     RemoveAll(key, hash, probe, deleted);
   }
 
-  void Free(autovector<ClockHandle>* deleted);
-
   // Tries to remove h from the hash table. If the attempt is successful,
   // the function hands over an exclusive ref to h.
   bool TryRemove(ClockHandle* h, autovector<ClockHandle>* deleted);
@@ -558,8 +571,13 @@ class ClockHandleTable {
   // success. Requires that the caller thread has no shared ref to h.
   bool SpinTryRemove(ClockHandle* h, autovector<ClockHandle>* deleted);
 
-  template <typename T>
-  void ApplyToEntriesRange(T func, uint32_t index_begin, uint32_t index_end,
+  // Call this function after an Insert, Remove, RemoveAll, TryRemove
+  // or SpinTryRemove. It frees the deleted values and updates the hash table
+  // metadata.
+  void Free(autovector<ClockHandle>* deleted);
+
+  void ApplyToEntriesRange(std::function<void(ClockHandle*)> func,
+                           uint32_t index_begin, uint32_t index_end,
                            bool apply_if_will_be_deleted) {
     for (uint32_t i = index_begin; i < index_end; i++) {
       ClockHandle* h = &array_[i];
@@ -573,18 +591,20 @@ class ClockHandleTable {
     }
   }
 
-  template <typename T>
-  void ConstApplyToEntriesRange(T func, uint32_t index_begin,
-                                uint32_t index_end,
+  void ConstApplyToEntriesRange(std::function<void(const ClockHandle*)> func,
+                                uint32_t index_begin, uint32_t index_end,
                                 bool apply_if_will_be_deleted) const {
     for (uint32_t i = index_begin; i < index_end; i++) {
       ClockHandle* h = &array_[i];
-      if (h->TryExclusiveRef()) {
+      // We take an external ref because we are handing over control
+      // to a user-defined function, and because the handle will not be
+      // modified.
+      if (h->TryExternalRef()) {
         if (h->IsElement() &&
             (apply_if_will_be_deleted || !h->WillBeDeleted())) {
           func(h);
         }
-        h->ReleaseExclusiveRef();
+        h->ReleaseExternalRef();
       }
     }
   }
@@ -601,6 +621,8 @@ class ClockHandleTable {
 
   size_t GetCapacity() const { return capacity_; }
 
+  void SetCapacity(size_t capacity) { capacity_ = capacity; }
+
   // Returns x mod 2^{length_bits_}.
   uint32_t ModTableSize(uint32_t x) { return x & length_bits_mask_; }
 
@@ -652,7 +674,7 @@ class ClockHandleTable {
   const uint32_t occupancy_limit_;
 
   // Maximum total charge of all elements stored in the table.
-  const size_t capacity_;
+  size_t capacity_;
 
   // We partition the following members into different cache lines
   // to avoid false sharing among Lookup, Release, Erase and Insert
@@ -745,10 +767,9 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShard {
 
  private:
   friend class ClockCache;
+  friend class ClockCacheTest;
 
-  // Free some space following strict clock policy until enough space
-  // to hold (usage_ + charge) is freed or there are no evictable elements.
-  void EvictFromClock(size_t charge, autovector<ClockHandle>* deleted);
+  ClockHandle* DetachedInsert(ClockHandle* h);
 
   // Returns the charge of a single handle.
   static size_t CalcEstimatedHandleCharge(
@@ -763,6 +784,9 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShard {
   // Whether to reject insertion if cache reaches its full capacity.
   std::atomic<bool> strict_capacity_limit_;
 
+  // Handles allocated separately from the table.
+  std::atomic<size_t> detached_usage_;
+
   ClockHandleTable table_;
 };  // class ClockCacheShard
 
@@ -797,6 +821,7 @@ class ClockCache
 
  private:
   ClockCacheShard* shards_ = nullptr;
+
   int num_shards_;
 };  // class ClockCache