@nxtedition/rocksdb 7.1.4 → 7.1.5

package/deps/rocksdb/rocksdb/cache/clock_cache.h

@@ -9,6 +9,8 @@
 
 #pragma once
 
+#include <sys/types.h>
+
 #include <array>
 #include <atomic>
 #include <cstdint>
@@ -28,6 +30,9 @@ namespace ROCKSDB_NAMESPACE {
 
 namespace clock_cache {
 
+// Forward declaration of friend class.
+class ClockCacheTest;
+
 // An experimental alternative to LRUCache, using a lock-free, open-addressed
 // hash table and clock eviction.
 
@@ -63,10 +68,10 @@ namespace clock_cache {
 //    can't be immediately deleted. In these cases, the flag will be later read
 //    and acted upon by the eviction algorithm. Importantly, WILL_BE_DELETED is
 //    used not only to defer deletions, but also as a barrier for external
-//    references: once WILL_BE_DELETED is set, lookups (which are the means to
-//    acquire new external references) will ignore the handle. For this reason,
-//    when WILL_BE_DELETED is set, we say the handle is invisible (and
-//    otherwise, that it's visible).
+//    references: once WILL_BE_DELETED is set, lookups (which are the most
+//    common way to acquire new external references) will ignore the handle.
+//    For this reason, when WILL_BE_DELETED is set, we say the handle is
+//    invisible (and, otherwise, that it's visible).
 //
 //
 // 3. HASHING AND COLLISION RESOLUTION
@@ -192,10 +197,10 @@ struct ClockHandle {
   size_t total_charge;
   std::array<char, kCacheKeySize> key_data;
 
-  static constexpr uint8_t kIsElementOffset = 1;
-  static constexpr uint8_t kClockPriorityOffset = 2;
-  static constexpr uint8_t kIsHitOffset = 4;
-  static constexpr uint8_t kCachePriorityOffset = 5;
+  static constexpr uint8_t kIsElementOffset = 0;
+  static constexpr uint8_t kClockPriorityOffset = 1;
+  static constexpr uint8_t kIsHitOffset = 3;
+  static constexpr uint8_t kCachePriorityOffset = 4;
 
   enum Flags : uint8_t {
     // Whether the slot is in use by an element.
@@ -252,9 +257,8 @@ struct ClockHandle {
     // Whether a thread has an exclusive reference to the slot.
     EXCLUSIVE_REF = uint32_t{1} << kExclusiveRefOffset,  // Bit 30
     // Whether the handle will be deleted soon. When this bit is set, new
-    // internal
-    // or external references to this handle stop being accepted.
-    // There is an exception: external references can be created from
+    // internal references to this handle stop being accepted.
+    // External references may still be granted---they can be created from
     // existing external references, or converting from existing internal
     // references.
     WILL_BE_DELETED = uint32_t{1} << kWillBeDeletedOffset  // Bit 31
@@ -274,6 +278,9 @@ struct ClockHandle {
 
   std::atomic<uint32_t> refs;
 
+  // True iff the handle is allocated separately from hash table.
+  bool detached;
+
   ClockHandle()
       : value(nullptr),
         deleter(nullptr),
@@ -281,7 +288,8 @@ struct ClockHandle {
         total_charge(0),
         flags(0),
         displacements(0),
-        refs(0) {
+        refs(0),
+        detached(false) {
     SetWillBeDeleted(false);
     SetIsElement(false);
     SetClockPriority(ClockPriority::NONE);
@@ -300,6 +308,7 @@ struct ClockHandle {
     value = other.value;
     deleter = other.deleter;
     key_data = other.key_data;
+    hash = other.hash;
     total_charge = other.total_charge;
   }
 
@@ -350,13 +359,13 @@ struct ClockHandle {
 
   // flags functions.
 
-  bool IsElement() const { return flags & IS_ELEMENT; }
+  bool IsElement() const { return flags & Flags::IS_ELEMENT; }
 
   void SetIsElement(bool is_element) {
     if (is_element) {
-      flags |= IS_ELEMENT;
+      flags |= Flags::IS_ELEMENT;
     } else {
-      flags &= static_cast<uint8_t>(~IS_ELEMENT);
+      flags &= static_cast<uint8_t>(~Flags::IS_ELEMENT);
     }
   }
 
@@ -400,6 +409,10 @@ struct ClockHandle {
     flags |= new_priority;
   }
 
+  bool IsDetached() { return detached; }
+
+  void SetDetached() { detached = true; }
+
   inline bool IsEmpty() const {
     return !this->IsElement() && this->displacements == 0;
   }
@@ -424,7 +437,9 @@ struct ClockHandle {
     }
   }
 
-  bool HasExternalRefs() const { return (refs & EXTERNAL_REFS) > 0; }
+  uint32_t ExternalRefs() const {
+    return (refs & EXTERNAL_REFS) >> kExternalRefsOffset;
+  }
 
   // Tries to take an internal ref. Returns true iff it succeeds.
   inline bool TryInternalRef() {
@@ -437,7 +452,7 @@ struct ClockHandle {
 
   // Tries to take an external ref. Returns true iff it succeeds.
   inline bool TryExternalRef() {
-    if (!((refs += kOneExternalRef) & (EXCLUSIVE_REF | WILL_BE_DELETED))) {
+    if (!((refs += kOneExternalRef) & EXCLUSIVE_REF)) {
       return true;
     }
     refs -= kOneExternalRef;
@@ -529,8 +544,8 @@ class ClockHandleTable {
   // Makes h non-evictable.
   void ClockOff(ClockHandle* h);
 
-  // Runs the clock eviction algorithm until there is enough space to
-  // insert an element with the given charge.
+  // Runs the clock eviction algorithm until usage_ + charge is at most
+  // capacity_.
   void ClockRun(size_t charge);
 
   // Remove h from the hash table. Requires an exclusive ref to h.
@@ -548,8 +563,6 @@ class ClockHandleTable {
     RemoveAll(key, hash, probe, deleted);
   }
 
-  void Free(autovector<ClockHandle>* deleted);
-
   // Tries to remove h from the hash table. If the attempt is successful,
   // the function hands over an exclusive ref to h.
   bool TryRemove(ClockHandle* h, autovector<ClockHandle>* deleted);
@@ -558,8 +571,13 @@ class ClockHandleTable {
   // success. Requires that the caller thread has no shared ref to h.
   bool SpinTryRemove(ClockHandle* h, autovector<ClockHandle>* deleted);
 
-  template <typename T>
-  void ApplyToEntriesRange(T func, uint32_t index_begin, uint32_t index_end,
+  // Call this function after an Insert, Remove, RemoveAll, TryRemove
+  // or SpinTryRemove. It frees the deleted values and updates the hash table
+  // metadata.
+  void Free(autovector<ClockHandle>* deleted);
+
+  void ApplyToEntriesRange(std::function<void(ClockHandle*)> func,
+                           uint32_t index_begin, uint32_t index_end,
                            bool apply_if_will_be_deleted) {
     for (uint32_t i = index_begin; i < index_end; i++) {
       ClockHandle* h = &array_[i];
@@ -573,18 +591,20 @@ class ClockHandleTable {
     }
   }
 
-  template <typename T>
-  void ConstApplyToEntriesRange(T func, uint32_t index_begin,
-                                uint32_t index_end,
+  void ConstApplyToEntriesRange(std::function<void(const ClockHandle*)> func,
+                                uint32_t index_begin, uint32_t index_end,
                                 bool apply_if_will_be_deleted) const {
     for (uint32_t i = index_begin; i < index_end; i++) {
       ClockHandle* h = &array_[i];
-      if (h->TryExclusiveRef()) {
+      // We take an external ref because we are handing over control
+      // to a user-defined function, and because the handle will not be
+      // modified.
+      if (h->TryExternalRef()) {
         if (h->IsElement() &&
             (apply_if_will_be_deleted || !h->WillBeDeleted())) {
           func(h);
         }
-        h->ReleaseExclusiveRef();
+        h->ReleaseExternalRef();
       }
     }
   }
@@ -601,6 +621,8 @@ class ClockHandleTable {
 
   size_t GetCapacity() const { return capacity_; }
 
+  void SetCapacity(size_t capacity) { capacity_ = capacity; }
+
   // Returns x mod 2^{length_bits_}.
   uint32_t ModTableSize(uint32_t x) { return x & length_bits_mask_; }
 
@@ -652,7 +674,7 @@ class ClockHandleTable {
   const uint32_t occupancy_limit_;
 
   // Maximum total charge of all elements stored in the table.
-  const size_t capacity_;
+  size_t capacity_;
 
   // We partition the following members into different cache lines
   // to avoid false sharing among Lookup, Release, Erase and Insert
@@ -745,10 +767,9 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShard {
 
  private:
   friend class ClockCache;
+  friend class ClockCacheTest;
 
-  // Free some space following strict clock policy until enough space
-  // to hold (usage_ + charge) is freed or there are no evictable elements.
-  void EvictFromClock(size_t charge, autovector<ClockHandle>* deleted);
+  ClockHandle* DetachedInsert(ClockHandle* h);
 
   // Returns the charge of a single handle.
   static size_t CalcEstimatedHandleCharge(
@@ -763,6 +784,9 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShard {
   // Whether to reject insertion if cache reaches its full capacity.
   std::atomic<bool> strict_capacity_limit_;
 
+  // Handles allocated separately from the table.
+  std::atomic<size_t> detached_usage_;
+
   ClockHandleTable table_;
 };  // class ClockCacheShard
 
@@ -797,6 +821,7 @@ class ClockCache
 
  private:
   ClockCacheShard* shards_ = nullptr;
+
   int num_shards_;
 };  // class ClockCache
 

package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.cc

@@ -5,6 +5,8 @@
 
 #include "cache/compressed_secondary_cache.h"
 
+#include <algorithm>
+#include <cstdint>
 #include <memory>
 
 #include "memory/memory_allocator.h"
@@ -13,15 +15,6 @@
 
 namespace ROCKSDB_NAMESPACE {
 
-namespace {
-
-void DeletionCallback(const Slice& /*key*/, void* obj) {
-  delete reinterpret_cast<CacheAllocationPtr*>(obj);
-  obj = nullptr;
-}
-
-}  // namespace
-
 CompressedSecondaryCache::CompressedSecondaryCache(
     size_t capacity, int num_shard_bits, bool strict_capacity_limit,
     double high_pri_pool_ratio,
@@ -49,26 +42,29 @@ std::unique_ptr<SecondaryCacheResultHandle> CompressedSecondaryCache::Lookup(
     return handle;
   }
 
-  CacheAllocationPtr* ptr =
-      reinterpret_cast<CacheAllocationPtr*>(cache_->Value(lru_handle));
-  void* value = nullptr;
-  size_t charge = 0;
-  Status s;
+  CacheValueChunk* handle_value =
+      reinterpret_cast<CacheValueChunk*>(cache_->Value(lru_handle));
+  size_t handle_value_charge{0};
+  CacheAllocationPtr merged_value =
+      MergeChunksIntoValue(handle_value, handle_value_charge);
 
+  Status s;
+  void* value{nullptr};
+  size_t charge{0};
   if (cache_options_.compression_type == kNoCompression) {
-    s = create_cb(ptr->get(), cache_->GetCharge(lru_handle), &value, &charge);
+    s = create_cb(merged_value.get(), handle_value_charge, &value, &charge);
   } else {
     UncompressionContext uncompression_context(cache_options_.compression_type);
     UncompressionInfo uncompression_info(uncompression_context,
                                          UncompressionDict::GetEmptyDict(),
                                          cache_options_.compression_type);
 
-    size_t uncompressed_size = 0;
+    size_t uncompressed_size{0};
     CacheAllocationPtr uncompressed;
-    uncompressed = UncompressData(
-        uncompression_info, (char*)ptr->get(), cache_->GetCharge(lru_handle),
-        &uncompressed_size, cache_options_.compress_format_version,
-        cache_options_.memory_allocator.get());
+    uncompressed = UncompressData(uncompression_info, (char*)merged_value.get(),
+                                  handle_value_charge, &uncompressed_size,
+                                  cache_options_.compress_format_version,
+                                  cache_options_.memory_allocator.get());
 
     if (!uncompressed) {
       cache_->Release(lru_handle, /* erase_if_last_ref */ true);
@@ -104,7 +100,7 @@ Status CompressedSecondaryCache::Insert(const Slice& key, void* value,
   if (cache_options_.compression_type != kNoCompression) {
     CompressionOptions compression_opts;
     CompressionContext compression_context(cache_options_.compression_type);
-    uint64_t sample_for_compression = 0;
+    uint64_t sample_for_compression{0};
     CompressionInfo compression_info(
         compression_opts, compression_context, CompressionDict::GetEmptyDict(),
         cache_options_.compression_type, sample_for_compression);
@@ -118,14 +114,12 @@ Status CompressedSecondaryCache::Insert(const Slice& key, void* value,
     }
 
     val = Slice(compressed_val);
-    size = compressed_val.size();
-    ptr = AllocateBlock(size, cache_options_.memory_allocator.get());
-    memcpy(ptr.get(), compressed_val.data(), size);
   }
 
-  CacheAllocationPtr* buf = new CacheAllocationPtr(std::move(ptr));
-
-  return cache_->Insert(key, buf, size, DeletionCallback);
+  size_t charge{0};
+  CacheValueChunk* value_chunks_head =
+      SplitValueIntoChunks(val, cache_options_.compression_type, charge);
+  return cache_->Insert(key, value_chunks_head, charge, DeletionCallback);
 }
 
 void CompressedSecondaryCache::Erase(const Slice& key) { cache_->Erase(key); }
@@ -133,7 +127,7 @@ void CompressedSecondaryCache::Erase(const Slice& key) { cache_->Erase(key); }
 std::string CompressedSecondaryCache::GetPrintableOptions() const {
   std::string ret;
   ret.reserve(20000);
-  const int kBufferSize = 200;
+  const int kBufferSize{200};
   char buffer[kBufferSize];
   ret.append(cache_->GetPrintableOptions());
   snprintf(buffer, kBufferSize, "    compression_type : %s\n",
@@ -145,6 +139,87 @@ std::string CompressedSecondaryCache::GetPrintableOptions() const {
   return ret;
 }
 
+CompressedSecondaryCache::CacheValueChunk*
+CompressedSecondaryCache::SplitValueIntoChunks(
+    const Slice& value, const CompressionType compression_type,
+    size_t& charge) {
+  assert(!value.empty());
+  const char* src_ptr = value.data();
+  size_t src_size{value.size()};
+
+  CacheValueChunk dummy_head = CacheValueChunk();
+  CacheValueChunk* current_chunk = &dummy_head;
+  // Do not split when value size is large or there is no compression.
+  size_t predicted_chunk_size{0};
+  size_t actual_chunk_size{0};
+  size_t tmp_size{0};
+  while (src_size > 0) {
+    predicted_chunk_size = sizeof(CacheValueChunk) - 1 + src_size;
+    auto upper =
+        std::upper_bound(malloc_bin_sizes_.begin(), malloc_bin_sizes_.end(),
+                         predicted_chunk_size);
+    // Do not split when value size is too small, too large, close to a bin
+    // size, or there is no compression.
+    if (upper == malloc_bin_sizes_.begin() ||
+        upper == malloc_bin_sizes_.end() ||
+        *upper - predicted_chunk_size < malloc_bin_sizes_.front() ||
+        compression_type == kNoCompression) {
+      tmp_size = predicted_chunk_size;
+    } else {
+      tmp_size = *(--upper);
+    }
+
+    CacheValueChunk* new_chunk =
+        reinterpret_cast<CacheValueChunk*>(new char[tmp_size]);
+    current_chunk->next = new_chunk;
+    current_chunk = current_chunk->next;
+    actual_chunk_size = tmp_size - sizeof(CacheValueChunk) + 1;
+    memcpy(current_chunk->data, src_ptr, actual_chunk_size);
+    current_chunk->size = actual_chunk_size;
+    src_ptr += actual_chunk_size;
+    src_size -= actual_chunk_size;
+    charge += tmp_size;
+  }
+  current_chunk->next = nullptr;
+
+  return dummy_head.next;
+}
+
+CacheAllocationPtr CompressedSecondaryCache::MergeChunksIntoValue(
+    const void* chunks_head, size_t& charge) {
+  const CacheValueChunk* head =
+      reinterpret_cast<const CacheValueChunk*>(chunks_head);
+  const CacheValueChunk* current_chunk = head;
+  charge = 0;
+  while (current_chunk != nullptr) {
+    charge += current_chunk->size;
+    current_chunk = current_chunk->next;
+  }
+
+  CacheAllocationPtr ptr =
+      AllocateBlock(charge, cache_options_.memory_allocator.get());
+  current_chunk = head;
+  size_t pos{0};
+  while (current_chunk != nullptr) {
+    memcpy(ptr.get() + pos, current_chunk->data, current_chunk->size);
+    pos += current_chunk->size;
+    current_chunk = current_chunk->next;
+  }
+
+  return ptr;
+}
+
+void CompressedSecondaryCache::DeletionCallback(const Slice& /*key*/,
+                                                void* obj) {
+  CacheValueChunk* chunks_head = reinterpret_cast<CacheValueChunk*>(obj);
+  while (chunks_head != nullptr) {
+    CacheValueChunk* tmp_chunk = chunks_head;
+    chunks_head = chunks_head->next;
+    tmp_chunk->Free();
+  }
+  obj = nullptr;
+}
+
 std::shared_ptr<SecondaryCache> NewCompressedSecondaryCache(
     size_t capacity, int num_shard_bits, bool strict_capacity_limit,
     double high_pri_pool_ratio,

package/deps/rocksdb/rocksdb/cache/compressed_secondary_cache.h

@@ -5,6 +5,8 @@
 
 #pragma once
 
+#include <array>
+#include <cstddef>
 #include <memory>
 
 #include "cache/lru_cache.h"
@@ -58,7 +60,7 @@ class CompressedSecondaryCache : public SecondaryCache {
       std::shared_ptr<MemoryAllocator> memory_allocator = nullptr,
       bool use_adaptive_mutex = kDefaultToAdaptiveMutex,
       CacheMetadataChargePolicy metadata_charge_policy =
-          kDontChargeCacheMetadata,
+          kDefaultCacheMetadataChargePolicy,
       CompressionType compression_type = CompressionType::kLZ4Compression,
       uint32_t compress_format_version = 2);
   virtual ~CompressedSecondaryCache() override;
@@ -79,6 +81,36 @@ class CompressedSecondaryCache : public SecondaryCache {
   std::string GetPrintableOptions() const override;
 
  private:
+  friend class CompressedSecondaryCacheTest;
+  static constexpr std::array<uint16_t, 33> malloc_bin_sizes_{
+      32,   64,   96,   128,  160,  192,  224,   256,   320,   384,   448,
+      512,  640,  768,  896,  1024, 1280, 1536,  1792,  2048,  2560,  3072,
+      3584, 4096, 5120, 6144, 7168, 8192, 10240, 12288, 14336, 16384, 32768};
+
+  struct CacheValueChunk {
+    // TODO try "CacheAllocationPtr next;".
+    CacheValueChunk* next;
+    size_t size;
+    // Beginning of the chunk data (MUST BE THE LAST FIELD IN THIS STRUCT!)
+    char data[1];
+
+    void Free() { delete[] reinterpret_cast<char*>(this); }
+  };
+
+  // Split value into chunks to better fit into jemalloc bins. The chunks
+  // are stored in CacheValueChunk and extra charge is needed for each chunk,
+  // so the cache charge is recalculated here.
+  CacheValueChunk* SplitValueIntoChunks(const Slice& value,
+                                        const CompressionType compression_type,
+                                        size_t& charge);
+
+  // After merging chunks, the extra charge for each chunk is removed, so
+  // the charge is recalculated.
+  CacheAllocationPtr MergeChunksIntoValue(const void* chunks_head,
+                                          size_t& charge);
+
+  // An implementation of Cache::DeleterFn.
+  static void DeletionCallback(const Slice& /*key*/, void* obj);
   std::shared_ptr<Cache> cache_;
   CompressedSecondaryCacheOptions cache_options_;
 };