@nxtedition/rocksdb 9.0.0 → 10.0.0

package/deps/rocksdb/rocksdb/cache/clock_cache.h

@@ -20,6 +20,7 @@
 #include "cache/sharded_cache.h"
 #include "port/lang.h"
 #include "port/malloc.h"
+#include "port/mmap.h"
 #include "port/port.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/secondary_cache.h"
@@ -39,24 +40,31 @@ class ClockCacheTest;
 //
 // Benefits
 // --------
-// * Fully lock free (no waits or spins) for efficiency under high concurrency
+// * Lock/wait free (no waits or spins) for efficiency under high concurrency
+//   * Fixed version (estimated_entry_charge > 0) is fully lock/wait free
+//   * Automatic version (estimated_entry_charge = 0) has rare waits among
+//     certain insertion or erase operations that involve the same very small
+//     set of entries.
 // * Optimized for hot path reads. For concurrency control, most Lookup() and
 // essentially all Release() are a single atomic add operation.
-// * Eviction on insertion is fully parallel and lock-free.
+// * Eviction on insertion is fully parallel.
 // * Uses a generalized + aging variant of CLOCK eviction that might outperform
 // LRU in some cases. (For background, see
 // https://en.wikipedia.org/wiki/Page_replacement_algorithm)
 //
 // Costs
 // -----
-// * Hash table is not resizable (for lock-free efficiency) so capacity is not
-// dynamically changeable. Rely on an estimated average value (block) size for
+// * FixedHyperClockCache (estimated_entry_charge > 0) - Hash table is not
+// resizable (for lock-free efficiency) so capacity is not dynamically
+// changeable. Rely on an estimated average value (block) size for
 // space+time efficiency. (See estimated_entry_charge option details.)
+// EXPERIMENTAL - This limitation is fixed in AutoHyperClockCache, activated
+// with estimated_entry_charge == 0.
 // * Insert usually does not (but might) overwrite a previous entry associated
-// with a cache key. This is OK for RocksDB uses of Cache.
+// with a cache key. This is OK for RocksDB uses of Cache, though it does mess
+// up our REDUNDANT block cache insertion statistics.
 // * Only supports keys of exactly 16 bytes, which is what RocksDB uses for
-// block cache (not row cache or table cache).
-// * SecondaryCache is not supported.
+// block cache (but not row cache or table cache).
 // * Cache priorities are less aggressively enforced. Unlike LRUCache, enough
 // transient LOW or BOTTOM priority items can evict HIGH priority entries that
 // are not referenced recently (or often) enough.
@@ -139,7 +147,8 @@ class ClockCacheTest;
 // * Empty - slot is not in use and unowned. All other metadata and data is
 // in an undefined state.
 // * Construction - slot is exclusively owned by one thread, the thread
-// successfully entering this state, for populating or freeing data.
+// successfully entering this state, for populating or freeing data
+// (de-construction, same state marker).
 // * Shareable (group) - slot holds an entry with counted references for
 // pinning and reading, including
 //   * Visible - slot holds an entry that can be returned by Lookup
@@ -187,15 +196,19 @@ class ClockCacheTest;
 // know from our "redundant" stats that overwrites are very rare for the block
 // cache, so we should not spend much to make them effective.
 //
-// So instead we Insert as soon as we find an empty slot in the probing
-// sequence without seeing an existing (visible) entry for the same key. This
-// way we only insert if we can improve the probing performance, and we don't
-// need to probe beyond our insert position, assuming we are willing to let
-// the previous entry for the same key die of old age (eventual eviction from
-// not being used). We can reach a similar state with concurrent insertions,
-// where one will pass over the other while it is "under construction."
-// This temporary duplication is acceptable for RocksDB block cache because
-// we know redundant insertion is rare.
+// FixedHyperClockCache: Instead we Insert as soon as we find an empty slot in
+// the probing sequence without seeing an existing (visible) entry for the same
+// key. This way we only insert if we can improve the probing performance, and
+// we don't need to probe beyond our insert position, assuming we are willing
+// to let the previous entry for the same key die of old age (eventual eviction
+// from not being used). We can reach a similar state with concurrent
+// insertions, where one will pass over the other while it is "under
+// construction." This temporary duplication is acceptable for RocksDB block
+// cache because we know redundant insertion is rare.
+// AutoHyperClockCache: Similar, except we only notice and return an existing
+// match if it is found in the search for a suitable empty slot (starting with
+// the same slot as the head pointer), not by following the existing chain of
+// entries. Insertions are always made to the head of the chain.
 //
 // Another problem to solve is what to return to the caller when we find an
 // existing entry whose probing position we cannot improve on, or when the
@@ -322,7 +335,6 @@ struct ClockHandle : public ClockHandleBasicData {
   // For setting the hit bit
   static constexpr uint8_t kHitBitShift = 2U * kCounterNumBits;
   static constexpr uint64_t kHitBitMask = uint64_t{1} << kHitBitShift;
-  ;
 
   // For reading or updating the state marker in meta word
   static constexpr uint8_t kStateShift = kHitBitShift + 1;
@@ -395,6 +407,8 @@ class BaseClockTable {
 
   uint32_t GetHashSeed() const { return hash_seed_; }
 
+  uint64_t GetYieldCount() const { return yield_count_.load(); }
+
   struct EvictionData {
     size_t freed_charge = 0;
     size_t freed_count = 0;
@@ -448,6 +462,9 @@ class BaseClockTable {
   // Clock algorithm sweep pointer.
   std::atomic<uint64_t> clock_pointer_{};
 
+  // Counter for number of times we yield to wait on another thread.
+  std::atomic<uint64_t> yield_count_{};
+
   // TODO: is this separation needed if we don't do background evictions?
   ALIGN_AS(CACHE_LINE_SIZE)
   // Number of elements in the table.
@@ -472,6 +489,10 @@ class BaseClockTable {
   const uint32_t& hash_seed_;
 };
 
+// Hash table for cache entries with size determined at creation time.
+// Uses open addressing and double hashing. Since entries cannot be moved,
+// the "displacements" count ensures probing sequences find entries even when
+// entries earlier in the probing sequence have been removed.
 class FixedHyperClockTable : public BaseClockTable {
  public:
   // Target size to be exactly a common cache line size (see static_assert in
@@ -626,11 +647,314 @@ class FixedHyperClockTable : public BaseClockTable {
   const std::unique_ptr<HandleImpl[]> array_;
 };  // class FixedHyperClockTable
 
-// Placeholder for future automatic table variant
-// For now, just use FixedHyperClockTable.
-class AutoHyperClockTable : public FixedHyperClockTable {
+// Hash table for cache entries that resizes automatically based on occupancy.
+// However, it depends on a contiguous memory region to grow into
+// incrementally, using linear hashing, so uses an anonymous mmap so that
+// only the used portion of the memory region is mapped to physical memory
+// (part of RSS).
+//
+// This table implementation uses the same "low-level protocol" for managing
+// the contens of an entry slot as FixedHyperClockTable does, captured in the
+// ClockHandle struct. The provides most of the essential data safety, but
+// AutoHyperClockTable is another "high-level protocol" for organizing entries
+// into a hash table, with automatic resizing.
+//
+// This implementation is not fully wait-free but we can call it "essentially
+// wait-free," and here's why. First, like FixedHyperClockCache, there is no
+// locking nor other forms of waiting at the cache or shard level. Also like
+// FixedHCC there is essentially an entry-level read-write lock implemented
+// with atomics, but our relaxed atomicity/consistency guarantees (e.g.
+// duplicate inserts are possible) mean we do not need to wait for entry
+// locking. Lookups, non-erasing Releases, and non-evicting non-growing Inserts
+// are all fully wait-free. Of course, these waits are not dependent on any
+// external factors such as I/O.
+//
+// For operations that remove entries from a chain or grow the table by
+// splitting a chain, there is a chain-level locking mechanism that we call a
+// "rewrite" lock, and the only waits are for these locks. On average, each
+// chain lock is relevant to < 2 entries each. (The average would be less than
+// one entry each, but we do not lock when there's no entry to remove or
+// migrate.) And a given thread can only hold two such chain locks at a time,
+// more typically just one. So in that sense alone, the waiting that does exist
+// is very localized.
+//
+// If we look closer at the operations utilizing that locking mechanism, we
+// can see why it's "essentially wait-free."
+// * Grow operations to increase the size of the table: each operation splits
+// an existing chain into two, and chains for splitting are chosen in table
+// order. Grow operations are fully parallel except for the chain locking, but
+// for one Grow operation to wait on another, it has to be feeding into the
+// other, which means the table has doubled in size already from other Grow
+// operations without the original one finishing. So Grow operations are very
+// low latency (unlike LRUCache doubling the table size in one operation) and
+// very parallelizeable. (We use some tricks to break up dependencies in
+// updating metadata on the usable size of the table.) And obviously Grow
+// operations are very rare after the initial population of the table.
+// * Evict operations (part of many Inserts): clock updates and evictions
+// sweep through the structure in table order, so like Grow operations,
+// parallel Evict can only wait on each other if an Evict has lingered (slept)
+// long enough that the clock pointer has wrapped around the entire structure.
+// * Random erasures (Erase, Release with erase_if_last_ref, etc.): these
+// operations are rare and not really considered performance critical.
+// Currently they're mostly used for removing placeholder cache entries, e.g.
+// for memory tracking, though that could use standalone entries instead to
+// avoid potential contention in table operations. It's possible that future
+// enhancements could pro-actively remove cache entries from obsolete files,
+// but that's not yet implemented.
+class AutoHyperClockTable : public BaseClockTable {
  public:
-  using FixedHyperClockTable::FixedHyperClockTable;
+  // Target size to be exactly a common cache line size (see static_assert in
+  // clock_cache.cc)
+  struct ALIGN_AS(64U) HandleImpl : public ClockHandle {
+    // To orgainize AutoHyperClockTable entries into a hash table while
+    // allowing the table size to grow without existing entries being moved,
+    // a version of chaining is used. Rather than being heap allocated (and
+    // incurring overheads to ensure memory safety) entries must go into
+    // Handles ("slots") in the pre-allocated array. To improve CPU cache
+    // locality, the chain head pointers are interleved with the entries;
+    // specifically, a Handle contains
+    // * A head pointer for a chain of entries with this "home" location.
+    // * A ClockHandle, for an entry that may or may not be in the chain
+    // starting from that head (but for performance ideally is on that
+    // chain).
+    // * A next pointer for the continuation of the chain containing this
+    // entry.
+    //
+    // The pointers are not raw pointers, but are indices into the array,
+    // and are decorated in two ways to help detect and recover from
+    // relevant concurrent modifications during Lookup, so that Lookup is
+    // fully wait-free:
+    // * Each "with_shift" pointer contains a shift count that indicates
+    // how many hash bits were used in chosing the home address for the
+    // chain--specifically the next entry in the chain.
+    // * The end of a chain is given a special "end" marker and refers back
+    // to the head of the chain.
+    //
+    // Why do we need shift on each pointer? To make Lookup wait-free, we need
+    // to be able to query a chain without missing anything, and preferably
+    // avoid synchronously double-checking the length_info. Without the shifts,
+    // there is a risk that we start down a chain and while paused on an entry
+    // that goes to a new home, we then follow the rest of the
+    // partially-migrated chain to see the shared ending with the old home, but
+    // for a time were following the chain for the new home, missing some
+    // entries for the old home.
+    //
+    // Why do we need the end of the chain to loop back? If Lookup pauses
+    // at an "under construction" entry, and sees that "next" is null after
+    // waking up, we need something to tell whether the "under construction"
+    // entry was freed and reused for another chain. Otherwise, we could
+    // miss entries still on the original chain due in the presence of a
+    // concurrent modification. Until an entry is fully erased from a chain,
+    // it is normal to see "under construction" entries on the chain, and it
+    // is not safe to read their hashed key without either a read reference
+    // on the entry or a rewrite lock on the chain.
+
+    // Marker in a "with_shift" head pointer for some thread owning writes
+    // to the chain structure (except for inserts), but only if not an
+    // "end" pointer. Also called the "rewrite lock."
+    static constexpr uint64_t kHeadLocked = uint64_t{1} << 7;
+
+    // Marker in a "with_shift" pointer for the end of a chain. Must also
+    // point back to the head of the chain (with end marker removed).
+    // Also includes the "locked" bit so that attempting to lock an empty
+    // chain has no effect (not needed, as the lock is only needed for
+    // removals).
+    static constexpr uint64_t kNextEndFlags = (uint64_t{1} << 6) | kHeadLocked;
+
+    static inline bool IsEnd(uint64_t next_with_shift) {
+      // Assuming certain values never used, suffices to check this one bit
+      constexpr auto kCheckBit = kNextEndFlags ^ kHeadLocked;
+      return next_with_shift & kCheckBit;
+    }
+
+    // Bottom bits to right shift away to get an array index from a
+    // "with_shift" pointer.
+    static constexpr int kNextShift = 8;
+
+    // A bit mask for the "shift" associated with each "with_shift" pointer.
+    // Always bottommost bits.
+    static constexpr int kShiftMask = 63;
+
+    // A marker for head_next_with_shift that indicates this HandleImpl is
+    // heap allocated (standalone) rather than in the table.
+    static constexpr uint64_t kStandaloneMarker = UINT64_MAX;
+
+    // A marker for head_next_with_shift indicating the head is not yet part
+    // of the usable table, or for chain_next_with_shift indicating that the
+    // entry is not present or is not yet part of a chain (must not be
+    // "shareable" state).
+    static constexpr uint64_t kUnusedMarker = 0;
+
+    // See above. The head pointer is logically independent of the rest of
+    // the entry, including the chain next pointer.
+    std::atomic<uint64_t> head_next_with_shift{kUnusedMarker};
+    std::atomic<uint64_t> chain_next_with_shift{kUnusedMarker};
+
+    // For supporting CreateStandalone and some fallback cases.
+    inline bool IsStandalone() const {
+      return head_next_with_shift.load(std::memory_order_acquire) ==
+             kStandaloneMarker;
+    }
+
+    inline void SetStandalone() {
+      head_next_with_shift.store(kStandaloneMarker, std::memory_order_release);
+    }
+  };  // struct HandleImpl
+
+  struct Opts {
+    explicit Opts(size_t _min_avg_value_size)
+        : min_avg_value_size(_min_avg_value_size) {}
+
+    explicit Opts(const HyperClockCacheOptions& opts) {
+      assert(opts.estimated_entry_charge == 0);
+      min_avg_value_size = opts.min_avg_entry_charge;
+    }
+    size_t min_avg_value_size;
+  };
+
+  AutoHyperClockTable(size_t capacity, bool strict_capacity_limit,
+                      CacheMetadataChargePolicy metadata_charge_policy,
+                      MemoryAllocator* allocator,
+                      const Cache::EvictionCallback* eviction_callback,
+                      const uint32_t* hash_seed, const Opts& opts);
+  ~AutoHyperClockTable();
+
+  // For BaseClockTable::Insert
+  struct InsertState {
+    uint64_t saved_length_info = 0;
+    size_t likely_empty_slot = 0;
+  };
+
+  void StartInsert(InsertState& state);
+
+  // Does initial check for whether there's hash table room for another
+  // inserted entry, possibly growing if needed. Returns true iff (after
+  // the call) there is room for the proposed number of entries.
+  bool GrowIfNeeded(size_t new_occupancy, InsertState& state);
+
+  HandleImpl* DoInsert(const ClockHandleBasicData& proto,
+                       uint64_t initial_countdown, bool take_ref,
+                       InsertState& state);
+
+  // Runs the clock eviction algorithm trying to reclaim at least
+  // requested_charge. Returns how much is evicted, which could be less
+  // if it appears impossible to evict the requested amount without blocking.
+  void Evict(size_t requested_charge, InsertState& state, EvictionData* data);
+
+  HandleImpl* Lookup(const UniqueId64x2& hashed_key);
+
+  bool Release(HandleImpl* handle, bool useful, bool erase_if_last_ref);
+
+  void Erase(const UniqueId64x2& hashed_key);
+
+  void EraseUnRefEntries();
+
+  size_t GetTableSize() const;
+
+  size_t GetOccupancyLimit() const;
+
+  const HandleImpl* HandlePtr(size_t idx) const { return &array_[idx]; }
+
+#ifndef NDEBUG
+  size_t& TEST_MutableOccupancyLimit() {
+    return *reinterpret_cast<size_t*>(&occupancy_limit_);
+  }
+
+  // Release N references
+  void TEST_ReleaseN(HandleImpl* handle, size_t n);
+#endif
+
+  // Maximum ratio of number of occupied slots to number of usable slots. The
+  // actual load factor should float pretty close to this number, which should
+  // be a nice space/time trade-off, though large swings in WriteBufferManager
+  // memory could lead to low (but very much safe) load factors (only after
+  // seeing high load factors). Linear hashing along with (modified) linear
+  // probing to find an available slot increases potential risks of high
+  // load factors, so are disallowed.
+  static constexpr double kMaxLoadFactor = 0.60;
+
+ private:  // functions
+  // Returns true iff increased usable length. Due to load factor
+  // considerations, GrowIfNeeded might call this more than once to make room
+  // for one more entry.
+  bool Grow(InsertState& state);
+
+  // Operational details of splitting a chain into two for Grow().
+  void SplitForGrow(size_t grow_home, size_t old_home, int old_shift);
+
+  // Takes an "under construction" entry and ensures it is no longer connected
+  // to its home chain (in preparaion for completing erasure and freeing the
+  // slot). Note that previous operations might have already noticed it being
+  // "under (de)construction" and removed it from its chain.
+  void Remove(HandleImpl* h);
+
+  // Try to take ownership of an entry and erase+remove it from the table.
+  // Returns true if successful. Could fail if
+  // * There are other references to the entry
+  // * Some other thread has exclusive ownership or has freed it.
+  bool TryEraseHandle(HandleImpl* h, bool holding_ref, bool mark_invisible);
+
+  // Calculates the appropriate maximum table size, for creating the memory
+  // mapping.
+  static size_t CalcMaxUsableLength(
+      size_t capacity, size_t min_avg_value_size,
+      CacheMetadataChargePolicy metadata_charge_policy);
+
+  // Shared helper function that implements removing entries from a chain
+  // with proper handling to ensure all existing data is seen even in the
+  // presence of concurrent insertions, etc. (See implementation.)
+  template <class OpData>
+  void PurgeImpl(OpData* op_data, size_t home = SIZE_MAX);
+
+  // An RAII wrapper for locking a chain of entries for removals. See
+  // implementation.
+  class ChainRewriteLock;
+
+  // Helper function for PurgeImpl while holding a ChainRewriteLock. See
+  // implementation.
+  template <class OpData>
+  void PurgeImplLocked(OpData* op_data, ChainRewriteLock& rewrite_lock,
+                       size_t home);
+
+  // Update length_info_ as much as possible without waiting, given a known
+  // usable (ready for inserts and lookups) grow_home. (Previous grow_homes
+  // might not be usable yet, but we can check if they are by looking at
+  // the corresponding old home.)
+  void CatchUpLengthInfoNoWait(size_t known_usable_grow_home);
+
+ private:  // data
+  // mmaped area holding handles
+  const TypedMemMapping<HandleImpl> array_;
+
+  // Metadata for table size under linear hashing.
+  //
+  // Lowest 8 bits are the minimum number of lowest hash bits to use
+  // ("min shift"). The upper 56 bits are a threshold. If that minumum number
+  // of bits taken from a hash value is < this threshold, then one more bit of
+  // hash value is taken and used.
+  //
+  // Other mechanisms (shift amounts on pointers) ensure complete availability
+  // of data already in the table even if a reader only sees a completely
+  // out-of-date version of this value. In the worst case, it could take
+  // log time to find the correct chain, but normally this value enables
+  // readers to find the correct chain on the first try.
+  //
+  // NOTES: length_info_ is only updated at the end of a Grow operation,
+  // so that waiting in Grow operations isn't done while entries are pinned
+  // for internal operation purposes. Thus, Lookup and Insert have to
+  // detect and support cases where length_info hasn't caught up to updated
+  // chains. Winning grow thread is the one that transitions
+  // head_next_with_shift from zeros. Grow threads can spin/yield wait for
+  // preconditions and postconditions to be met.
+  std::atomic<uint64_t> length_info_;
+
+  // An already-computed version of the usable length times the max load
+  // factor. Could be slightly out of date but GrowIfNeeded()/Grow() handle
+  // that internally.
+  std::atomic<size_t> occupancy_limit_;
+
+  // See explanation in AutoHyperClockTable::Evict
+  std::atomic<size_t> clock_pointer_mask_;
 };  // class AutoHyperClockTable
 
 // A single shard of sharded cache.
@@ -785,7 +1109,6 @@ class FixedHyperClockCache
       const std::shared_ptr<Logger>& /*info_log*/) const override;
 };  // class FixedHyperClockCache
 
-// Placeholder for future automatic HCC variant
 class AutoHyperClockCache
 #ifdef NDEBUG
     final
@@ -795,6 +1118,9 @@ class AutoHyperClockCache
   using BaseHyperClockCache::BaseHyperClockCache;
 
   const char* Name() const override { return "AutoHyperClockCache"; }
+
+  void ReportProblems(
+      const std::shared_ptr<Logger>& /*info_log*/) const override;
 };  // class AutoHyperClockCache
 
 }  // namespace clock_cache