@nxtedition/rocksdb 8.2.8 → 9.0.1

package/deps/rocksdb/rocksdb/cache/clock_cache.h

@@ -20,16 +20,19 @@
 #include "cache/sharded_cache.h"
 #include "port/lang.h"
 #include "port/malloc.h"
+#include "port/mmap.h"
 #include "port/port.h"
 #include "rocksdb/cache.h"
 #include "rocksdb/secondary_cache.h"
 #include "util/autovector.h"
+#include "util/math.h"
 
 namespace ROCKSDB_NAMESPACE {
 
 namespace clock_cache {
 
 // Forward declaration of friend class.
+template <class ClockCache>
 class ClockCacheTest;
 
 // HyperClockCache is an alternative to LRUCache specifically tailored for
@@ -37,24 +40,31 @@ class ClockCacheTest;
 //
 // Benefits
 // --------
-// * Fully lock free (no waits or spins) for efficiency under high concurrency
+// * Lock/wait free (no waits or spins) for efficiency under high concurrency
+//   * Fixed version (estimated_entry_charge > 0) is fully lock/wait free
+//   * Automatic version (estimated_entry_charge = 0) has rare waits among
+//     certain insertion or erase operations that involve the same very small
+//     set of entries.
 // * Optimized for hot path reads. For concurrency control, most Lookup() and
 // essentially all Release() are a single atomic add operation.
-// * Eviction on insertion is fully parallel and lock-free.
+// * Eviction on insertion is fully parallel.
 // * Uses a generalized + aging variant of CLOCK eviction that might outperform
 // LRU in some cases. (For background, see
 // https://en.wikipedia.org/wiki/Page_replacement_algorithm)
 //
 // Costs
 // -----
-// * Hash table is not resizable (for lock-free efficiency) so capacity is not
-// dynamically changeable. Rely on an estimated average value (block) size for
+// * FixedHyperClockCache (estimated_entry_charge > 0) - Hash table is not
+// resizable (for lock-free efficiency) so capacity is not dynamically
+// changeable. Rely on an estimated average value (block) size for
 // space+time efficiency. (See estimated_entry_charge option details.)
+// EXPERIMENTAL - This limitation is fixed in AutoHyperClockCache, activated
+// with estimated_entry_charge == 0.
 // * Insert usually does not (but might) overwrite a previous entry associated
-// with a cache key. This is OK for RocksDB uses of Cache.
+// with a cache key. This is OK for RocksDB uses of Cache, though it does mess
+// up our REDUNDANT block cache insertion statistics.
 // * Only supports keys of exactly 16 bytes, which is what RocksDB uses for
-// block cache (not row cache or table cache).
-// * SecondaryCache is not supported.
+// block cache (but not row cache or table cache).
 // * Cache priorities are less aggressively enforced. Unlike LRUCache, enough
 // transient LOW or BOTTOM priority items can evict HIGH priority entries that
 // are not referenced recently (or often) enough.
@@ -137,7 +147,8 @@ class ClockCacheTest;
 // * Empty - slot is not in use and unowned. All other metadata and data is
 // in an undefined state.
 // * Construction - slot is exclusively owned by one thread, the thread
-// successfully entering this state, for populating or freeing data.
+// successfully entering this state, for populating or freeing data
+// (de-construction, same state marker).
 // * Shareable (group) - slot holds an entry with counted references for
 // pinning and reading, including
 //   * Visible - slot holds an entry that can be returned by Lookup
@@ -185,15 +196,19 @@ class ClockCacheTest;
 // know from our "redundant" stats that overwrites are very rare for the block
 // cache, so we should not spend much to make them effective.
 //
-// So instead we Insert as soon as we find an empty slot in the probing
-// sequence without seeing an existing (visible) entry for the same key. This
-// way we only insert if we can improve the probing performance, and we don't
-// need to probe beyond our insert position, assuming we are willing to let
-// the previous entry for the same key die of old age (eventual eviction from
-// not being used). We can reach a similar state with concurrent insertions,
-// where one will pass over the other while it is "under construction."
-// This temporary duplication is acceptable for RocksDB block cache because
-// we know redundant insertion is rare.
+// FixedHyperClockCache: Instead we Insert as soon as we find an empty slot in
+// the probing sequence without seeing an existing (visible) entry for the same
+// key. This way we only insert if we can improve the probing performance, and
+// we don't need to probe beyond our insert position, assuming we are willing
+// to let the previous entry for the same key die of old age (eventual eviction
+// from not being used). We can reach a similar state with concurrent
+// insertions, where one will pass over the other while it is "under
+// construction." This temporary duplication is acceptable for RocksDB block
+// cache because we know redundant insertion is rare.
+// AutoHyperClockCache: Similar, except we only notice and return an existing
+// match if it is found in the search for a suitable empty slot (starting with
+// the same slot as the head pointer), not by following the existing chain of
+// entries. Insertions are always made to the head of the chain.
 //
 // Another problem to solve is what to return to the caller when we find an
 // existing entry whose probing position we cannot improve on, or when the
@@ -281,29 +296,6 @@ class ClockCacheTest;
 
 // ----------------------------------------------------------------------- //
 
-// The load factor p is a real number in (0, 1) such that at all
-// times at most a fraction p of all slots, without counting tombstones,
-// are occupied by elements. This means that the probability that a random
-// probe hits an occupied slot is at most p, and thus at most 1/p probes
-// are required on average. For example, p = 70% implies that between 1 and 2
-// probes are needed on average (bear in mind that this reasoning doesn't
-// consider the effects of clustering over time, which should be negligible
-// with double hashing).
-// Because the size of the hash table is always rounded up to the next
-// power of 2, p is really an upper bound on the actual load factor---the
-// actual load factor is anywhere between p/2 and p. This is a bit wasteful,
-// but bear in mind that slots only hold metadata, not actual values.
-// Since space cost is dominated by the values (the LSM blocks),
-// overprovisioning the table with metadata only increases the total cache space
-// usage by a tiny fraction.
-constexpr double kLoadFactor = 0.7;
-
-// The user can exceed kLoadFactor if the sizes of the inserted values don't
-// match estimated_value_size, or in some rare cases with
-// strict_capacity_limit == false. To avoid degenerate performance, we set a
-// strict upper bound on the load factor.
-constexpr double kStrictLoadFactor = 0.84;
-
 struct ClockHandleBasicData {
   Cache::ObjectPtr value = nullptr;
   const Cache::CacheItemHelper* helper = nullptr;
@@ -326,7 +318,7 @@ struct ClockHandle : public ClockHandleBasicData {
   // state of the handle. The meta word looks like this:
   // low bits                                                     high bits
   // -----------------------------------------------------------------------
-  // | acquire counter          | release counter           | state marker |
+  // | acquire counter      | release counter     | hit bit | state marker |
   // -----------------------------------------------------------------------
 
   // For reading or updating counters in meta word.
@@ -340,8 +332,12 @@ struct ClockHandle : public ClockHandleBasicData {
   static constexpr uint64_t kReleaseIncrement = uint64_t{1}
                                                 << kReleaseCounterShift;
 
+  // For setting the hit bit
+  static constexpr uint8_t kHitBitShift = 2U * kCounterNumBits;
+  static constexpr uint64_t kHitBitMask = uint64_t{1} << kHitBitShift;
+
   // For reading or updating the state marker in meta word
-  static constexpr uint8_t kStateShift = 2U * kCounterNumBits;
+  static constexpr uint8_t kStateShift = kHitBitShift + 1;
 
   // Bits contribution to state marker.
   // Occupied means any state other than empty
@@ -371,14 +367,133 @@ struct ClockHandle : public ClockHandleBasicData {
   static constexpr uint8_t kMaxCountdown = kHighCountdown;
   // TODO: make these coundown values tuning parameters for eviction?
 
-  // See above
-  std::atomic<uint64_t> meta{};
-
-  // Anticipating use for SecondaryCache support
-  void* reserved_for_future_use = nullptr;
+  // See above. Mutable for read reference counting.
+  mutable std::atomic<uint64_t> meta{};
 };  // struct ClockHandle
 
-class HyperClockTable {
+class BaseClockTable {
+ public:
+  BaseClockTable(CacheMetadataChargePolicy metadata_charge_policy,
+                 MemoryAllocator* allocator,
+                 const Cache::EvictionCallback* eviction_callback,
+                 const uint32_t* hash_seed)
+      : metadata_charge_policy_(metadata_charge_policy),
+        allocator_(allocator),
+        eviction_callback_(*eviction_callback),
+        hash_seed_(*hash_seed) {}
+
+  template <class Table>
+  typename Table::HandleImpl* CreateStandalone(ClockHandleBasicData& proto,
+                                               size_t capacity,
+                                               bool strict_capacity_limit,
+                                               bool allow_uncharged);
+
+  template <class Table>
+  Status Insert(const ClockHandleBasicData& proto,
+                typename Table::HandleImpl** handle, Cache::Priority priority,
+                size_t capacity, bool strict_capacity_limit);
+
+  void Ref(ClockHandle& handle);
+
+  size_t GetOccupancy() const {
+    return occupancy_.load(std::memory_order_relaxed);
+  }
+
+  size_t GetUsage() const { return usage_.load(std::memory_order_relaxed); }
+
+  size_t GetStandaloneUsage() const {
+    return standalone_usage_.load(std::memory_order_relaxed);
+  }
+
+  uint32_t GetHashSeed() const { return hash_seed_; }
+
+  uint64_t GetYieldCount() const { return yield_count_.load(); }
+
+  struct EvictionData {
+    size_t freed_charge = 0;
+    size_t freed_count = 0;
+  };
+
+  void TrackAndReleaseEvictedEntry(ClockHandle* h, EvictionData* data);
+
+#ifndef NDEBUG
+  // Acquire N references
+  void TEST_RefN(ClockHandle& handle, size_t n);
+  // Helper for TEST_ReleaseN
+  void TEST_ReleaseNMinus1(ClockHandle* handle, size_t n);
+#endif
+
+ private:  // fns
+  // Creates a "standalone" handle for returning from an Insert operation that
+  // cannot be completed by actually inserting into the table.
+  // Updates `standalone_usage_` but not `usage_` nor `occupancy_`.
+  template <class HandleImpl>
+  HandleImpl* StandaloneInsert(const ClockHandleBasicData& proto);
+
+  // Helper for updating `usage_` for new entry with given `total_charge`
+  // and evicting if needed under strict_capacity_limit=true rules. This
+  // means the operation might fail with Status::MemoryLimit. If
+  // `need_evict_for_occupancy`, then eviction of at least one entry is
+  // required, and the operation should fail if not possible.
+  // NOTE: Otherwise, occupancy_ is not managed in this function
+  template <class Table>
+  Status ChargeUsageMaybeEvictStrict(size_t total_charge, size_t capacity,
+                                     bool need_evict_for_occupancy,
+                                     typename Table::InsertState& state);
+
+  // Helper for updating `usage_` for new entry with given `total_charge`
+  // and evicting if needed under strict_capacity_limit=false rules. This
+  // means that updating `usage_` always succeeds even if forced to exceed
+  // capacity. If `need_evict_for_occupancy`, then eviction of at least one
+  // entry is required, and the operation should return false if such eviction
+  // is not possible. `usage_` is not updated in that case. Otherwise, returns
+  // true, indicating success.
+  // NOTE: occupancy_ is not managed in this function
+  template <class Table>
+  bool ChargeUsageMaybeEvictNonStrict(size_t total_charge, size_t capacity,
+                                      bool need_evict_for_occupancy,
+                                      typename Table::InsertState& state);
+
+ protected:  // data
+  // We partition the following members into different cache lines
+  // to avoid false sharing among Lookup, Release, Erase and Insert
+  // operations in ClockCacheShard.
+
+  // Clock algorithm sweep pointer.
+  std::atomic<uint64_t> clock_pointer_{};
+
+  // Counter for number of times we yield to wait on another thread.
+  std::atomic<uint64_t> yield_count_{};
+
+  // TODO: is this separation needed if we don't do background evictions?
+  ALIGN_AS(CACHE_LINE_SIZE)
+  // Number of elements in the table.
+  std::atomic<size_t> occupancy_{};
+
+  // Memory usage by entries tracked by the cache (including standalone)
+  std::atomic<size_t> usage_{};
+
+  // Part of usage by standalone entries (not in table)
+  std::atomic<size_t> standalone_usage_{};
+
+  ALIGN_AS(CACHE_LINE_SIZE)
+  const CacheMetadataChargePolicy metadata_charge_policy_;
+
+  // From Cache, for deleter
+  MemoryAllocator* const allocator_;
+
+  // A reference to Cache::eviction_callback_
+  const Cache::EvictionCallback& eviction_callback_;
+
+  // A reference to ShardedCacheBase::hash_seed_
+  const uint32_t& hash_seed_;
+};
+
+// Hash table for cache entries with size determined at creation time.
+// Uses open addressing and double hashing. Since entries cannot be moved,
+// the "displacements" count ensures probing sequences find entries even when
+// entries earlier in the probing sequence have been removed.
+class FixedHyperClockTable : public BaseClockTable {
  public:
   // Target size to be exactly a common cache line size (see static_assert in
   // clock_cache.cc)
@@ -400,88 +515,104 @@ class HyperClockTable {
   };  // struct HandleImpl
 
   struct Opts {
+    explicit Opts(size_t _estimated_value_size)
+        : estimated_value_size(_estimated_value_size) {}
+    explicit Opts(const HyperClockCacheOptions& opts) {
+      assert(opts.estimated_entry_charge > 0);
+      estimated_value_size = opts.estimated_entry_charge;
+    }
     size_t estimated_value_size;
   };
 
-  HyperClockTable(size_t capacity, bool strict_capacity_limit,
-                  CacheMetadataChargePolicy metadata_charge_policy,
-                  MemoryAllocator* allocator,
-                  const Cache::EvictionCallback* eviction_callback,
-                  const uint32_t* hash_seed, const Opts& opts);
-  ~HyperClockTable();
+  FixedHyperClockTable(size_t capacity, bool strict_capacity_limit,
+                       CacheMetadataChargePolicy metadata_charge_policy,
+                       MemoryAllocator* allocator,
+                       const Cache::EvictionCallback* eviction_callback,
+                       const uint32_t* hash_seed, const Opts& opts);
+  ~FixedHyperClockTable();
+
+  // For BaseClockTable::Insert
+  struct InsertState {};
 
-  Status Insert(const ClockHandleBasicData& proto, HandleImpl** handle,
-                Cache::Priority priority, size_t capacity,
-                bool strict_capacity_limit);
+  void StartInsert(InsertState& state);
 
-  HandleImpl* CreateStandalone(ClockHandleBasicData& proto, size_t capacity,
-                               bool strict_capacity_limit,
-                               bool allow_uncharged);
+  // Returns true iff there is room for the proposed number of entries.
+  bool GrowIfNeeded(size_t new_occupancy, InsertState& state);
+
+  HandleImpl* DoInsert(const ClockHandleBasicData& proto,
+                       uint64_t initial_countdown, bool take_ref,
+                       InsertState& state);
+
+  // Runs the clock eviction algorithm trying to reclaim at least
+  // requested_charge. Returns how much is evicted, which could be less
+  // if it appears impossible to evict the requested amount without blocking.
+  void Evict(size_t requested_charge, InsertState& state, EvictionData* data);
 
   HandleImpl* Lookup(const UniqueId64x2& hashed_key);
 
   bool Release(HandleImpl* handle, bool useful, bool erase_if_last_ref);
 
-  void Ref(HandleImpl& handle);
-
   void Erase(const UniqueId64x2& hashed_key);
 
-  void ConstApplyToEntriesRange(std::function<void(const HandleImpl&)> func,
-                                size_t index_begin, size_t index_end,
-                                bool apply_if_will_be_deleted) const;
-
   void EraseUnRefEntries();
 
   size_t GetTableSize() const { return size_t{1} << length_bits_; }
 
-  int GetLengthBits() const { return length_bits_; }
-
-  size_t GetOccupancy() const {
-    return occupancy_.load(std::memory_order_relaxed);
-  }
-
   size_t GetOccupancyLimit() const { return occupancy_limit_; }
 
-  size_t GetUsage() const { return usage_.load(std::memory_order_relaxed); }
+  const HandleImpl* HandlePtr(size_t idx) const { return &array_[idx]; }
 
-  size_t GetStandaloneUsage() const {
-    return standalone_usage_.load(std::memory_order_relaxed);
+#ifndef NDEBUG
+  size_t& TEST_MutableOccupancyLimit() {
+    return const_cast<size_t&>(occupancy_limit_);
   }
 
-  uint32_t GetHashSeed() const { return hash_seed_; }
-
-  // Acquire/release N references
-  void TEST_RefN(HandleImpl& handle, size_t n);
+  // Release N references
   void TEST_ReleaseN(HandleImpl* handle, size_t n);
+#endif
+
+  // The load factor p is a real number in (0, 1) such that at all
+  // times at most a fraction p of all slots, without counting tombstones,
+  // are occupied by elements. This means that the probability that a random
+  // probe hits an occupied slot is at most p, and thus at most 1/p probes
+  // are required on average. For example, p = 70% implies that between 1 and 2
+  // probes are needed on average (bear in mind that this reasoning doesn't
+  // consider the effects of clustering over time, which should be negligible
+  // with double hashing).
+  // Because the size of the hash table is always rounded up to the next
+  // power of 2, p is really an upper bound on the actual load factor---the
+  // actual load factor is anywhere between p/2 and p. This is a bit wasteful,
+  // but bear in mind that slots only hold metadata, not actual values.
+  // Since space cost is dominated by the values (the LSM blocks),
+  // overprovisioning the table with metadata only increases the total cache
+  // space usage by a tiny fraction.
+  static constexpr double kLoadFactor = 0.7;
+
+  // The user can exceed kLoadFactor if the sizes of the inserted values don't
+  // match estimated_value_size, or in some rare cases with
+  // strict_capacity_limit == false. To avoid degenerate performance, we set a
+  // strict upper bound on the load factor.
+  static constexpr double kStrictLoadFactor = 0.84;
 
  private:  // functions
   // Returns x mod 2^{length_bits_}.
   inline size_t ModTableSize(uint64_t x) {
-    return static_cast<size_t>(x) & length_bits_mask_;
+    return BitwiseAnd(x, length_bits_mask_);
   }
 
-  // Runs the clock eviction algorithm trying to reclaim at least
-  // requested_charge. Returns how much is evicted, which could be less
-  // if it appears impossible to evict the requested amount without blocking.
-  inline void Evict(size_t requested_charge, size_t* freed_charge,
-                    size_t* freed_count);
-
-  // Returns the first slot in the probe sequence, starting from the given
-  // probe number, with a handle e such that match(e) is true. At every
-  // step, the function first tests whether match(e) holds. If this is false,
-  // it evaluates abort(e) to decide whether the search should be aborted,
-  // and in the affirmative returns -1. For every handle e probed except
-  // the last one, the function runs update(e).
-  // The probe parameter is modified as follows. We say a probe to a handle
-  // e is aborting if match(e) is false and abort(e) is true. Then the final
-  // value of probe is one more than the last non-aborting probe during the
-  // call. This is so that that the variable can be used to keep track of
-  // progress across consecutive calls to FindSlot.
+  // Returns the first slot in the probe sequence with a handle e such that
+  // match_fn(e) is true. At every step, the function first tests whether
+  // match_fn(e) holds. If this is false, it evaluates abort_fn(e) to decide
+  // whether the search should be aborted, and if so, FindSlot immediately
+  // returns nullptr. For every handle e that is not a match and not aborted,
+  // FindSlot runs update_fn(e, is_last) where is_last is set to true iff that
+  // slot will be the last probed because the next would cycle back to the first
+  // slot probed. This function uses templates instead of std::function to
+  // minimize the risk of heap-allocated closures being created.
+  template <typename MatchFn, typename AbortFn, typename UpdateFn>
   inline HandleImpl* FindSlot(const UniqueId64x2& hashed_key,
-                              std::function<bool(HandleImpl*)> match,
-                              std::function<bool(HandleImpl*)> stop,
-                              std::function<void(HandleImpl*)> update,
-                              size_t& probe);
+                              const MatchFn& match_fn, const AbortFn& abort_fn,
+                              const UpdateFn& update_fn);
 
   // Re-decrement all displacements in probe path starting from beginning
   // until (not including) the given handle
@@ -494,33 +625,6 @@ class HyperClockTable {
   // before releasing it so that it can be provided to this function.
   inline void ReclaimEntryUsage(size_t total_charge);
 
-  // Helper for updating `usage_` for new entry with given `total_charge`
-  // and evicting if needed under strict_capacity_limit=true rules. This
-  // means the operation might fail with Status::MemoryLimit. If
-  // `need_evict_for_occupancy`, then eviction of at least one entry is
-  // required, and the operation should fail if not possible.
-  // NOTE: Otherwise, occupancy_ is not managed in this function
-  inline Status ChargeUsageMaybeEvictStrict(size_t total_charge,
-                                            size_t capacity,
-                                            bool need_evict_for_occupancy);
-
-  // Helper for updating `usage_` for new entry with given `total_charge`
-  // and evicting if needed under strict_capacity_limit=false rules. This
-  // means that updating `usage_` always succeeds even if forced to exceed
-  // capacity. If `need_evict_for_occupancy`, then eviction of at least one
-  // entry is required, and the operation should return false if such eviction
-  // is not possible. `usage_` is not updated in that case. Otherwise, returns
-  // true, indicating success.
-  // NOTE: occupancy_ is not managed in this function
-  inline bool ChargeUsageMaybeEvictNonStrict(size_t total_charge,
-                                             size_t capacity,
-                                             bool need_evict_for_occupancy);
-
-  // Creates a "standalone" handle for returning from an Insert operation that
-  // cannot be completed by actually inserting into the table.
-  // Updates `standalone_usage_` but not `usage_` nor `occupancy_`.
-  inline HandleImpl* StandaloneInsert(const ClockHandleBasicData& proto);
-
   MemoryAllocator* GetAllocator() const { return allocator_; }
 
   // Returns the number of bits used to hash an element in the hash
@@ -541,39 +645,323 @@ class HyperClockTable {
 
   // Array of slots comprising the hash table.
   const std::unique_ptr<HandleImpl[]> array_;
+};  // class FixedHyperClockTable
 
-  // From Cache, for deleter
-  MemoryAllocator* const allocator_;
+// Hash table for cache entries that resizes automatically based on occupancy.
+// However, it depends on a contiguous memory region to grow into
+// incrementally, using linear hashing, so uses an anonymous mmap so that
+// only the used portion of the memory region is mapped to physical memory
+// (part of RSS).
+//
+// This table implementation uses the same "low-level protocol" for managing
+// the contens of an entry slot as FixedHyperClockTable does, captured in the
+// ClockHandle struct. The provides most of the essential data safety, but
+// AutoHyperClockTable is another "high-level protocol" for organizing entries
+// into a hash table, with automatic resizing.
+//
+// This implementation is not fully wait-free but we can call it "essentially
+// wait-free," and here's why. First, like FixedHyperClockCache, there is no
+// locking nor other forms of waiting at the cache or shard level. Also like
+// FixedHCC there is essentially an entry-level read-write lock implemented
+// with atomics, but our relaxed atomicity/consistency guarantees (e.g.
+// duplicate inserts are possible) mean we do not need to wait for entry
+// locking. Lookups, non-erasing Releases, and non-evicting non-growing Inserts
+// are all fully wait-free. Of course, these waits are not dependent on any
+// external factors such as I/O.
+//
+// For operations that remove entries from a chain or grow the table by
+// splitting a chain, there is a chain-level locking mechanism that we call a
+// "rewrite" lock, and the only waits are for these locks. On average, each
+// chain lock is relevant to < 2 entries each. (The average would be less than
+// one entry each, but we do not lock when there's no entry to remove or
+// migrate.) And a given thread can only hold two such chain locks at a time,
+// more typically just one. So in that sense alone, the waiting that does exist
+// is very localized.
+//
+// If we look closer at the operations utilizing that locking mechanism, we
+// can see why it's "essentially wait-free."
+// * Grow operations to increase the size of the table: each operation splits
+// an existing chain into two, and chains for splitting are chosen in table
+// order. Grow operations are fully parallel except for the chain locking, but
+// for one Grow operation to wait on another, it has to be feeding into the
+// other, which means the table has doubled in size already from other Grow
+// operations without the original one finishing. So Grow operations are very
+// low latency (unlike LRUCache doubling the table size in one operation) and
+// very parallelizeable. (We use some tricks to break up dependencies in
+// updating metadata on the usable size of the table.) And obviously Grow
+// operations are very rare after the initial population of the table.
+// * Evict operations (part of many Inserts): clock updates and evictions
+// sweep through the structure in table order, so like Grow operations,
+// parallel Evict can only wait on each other if an Evict has lingered (slept)
+// long enough that the clock pointer has wrapped around the entire structure.
+// * Random erasures (Erase, Release with erase_if_last_ref, etc.): these
+// operations are rare and not really considered performance critical.
+// Currently they're mostly used for removing placeholder cache entries, e.g.
+// for memory tracking, though that could use standalone entries instead to
+// avoid potential contention in table operations. It's possible that future
+// enhancements could pro-actively remove cache entries from obsolete files,
+// but that's not yet implemented.
+class AutoHyperClockTable : public BaseClockTable {
+ public:
+  // Target size to be exactly a common cache line size (see static_assert in
+  // clock_cache.cc)
+  struct ALIGN_AS(64U) HandleImpl : public ClockHandle {
+    // To orgainize AutoHyperClockTable entries into a hash table while
+    // allowing the table size to grow without existing entries being moved,
+    // a version of chaining is used. Rather than being heap allocated (and
+    // incurring overheads to ensure memory safety) entries must go into
+    // Handles ("slots") in the pre-allocated array. To improve CPU cache
+    // locality, the chain head pointers are interleved with the entries;
+    // specifically, a Handle contains
+    // * A head pointer for a chain of entries with this "home" location.
+    // * A ClockHandle, for an entry that may or may not be in the chain
+    // starting from that head (but for performance ideally is on that
+    // chain).
+    // * A next pointer for the continuation of the chain containing this
+    // entry.
+    //
+    // The pointers are not raw pointers, but are indices into the array,
+    // and are decorated in two ways to help detect and recover from
+    // relevant concurrent modifications during Lookup, so that Lookup is
+    // fully wait-free:
+    // * Each "with_shift" pointer contains a shift count that indicates
+    // how many hash bits were used in chosing the home address for the
+    // chain--specifically the next entry in the chain.
+    // * The end of a chain is given a special "end" marker and refers back
+    // to the head of the chain.
+    //
+    // Why do we need shift on each pointer? To make Lookup wait-free, we need
+    // to be able to query a chain without missing anything, and preferably
+    // avoid synchronously double-checking the length_info. Without the shifts,
+    // there is a risk that we start down a chain and while paused on an entry
+    // that goes to a new home, we then follow the rest of the
+    // partially-migrated chain to see the shared ending with the old home, but
+    // for a time were following the chain for the new home, missing some
+    // entries for the old home.
+    //
+    // Why do we need the end of the chain to loop back? If Lookup pauses
+    // at an "under construction" entry, and sees that "next" is null after
+    // waking up, we need something to tell whether the "under construction"
+    // entry was freed and reused for another chain. Otherwise, we could
+    // miss entries still on the original chain due in the presence of a
+    // concurrent modification. Until an entry is fully erased from a chain,
+    // it is normal to see "under construction" entries on the chain, and it
+    // is not safe to read their hashed key without either a read reference
+    // on the entry or a rewrite lock on the chain.
+
+    // Marker in a "with_shift" head pointer for some thread owning writes
+    // to the chain structure (except for inserts), but only if not an
+    // "end" pointer. Also called the "rewrite lock."
+    static constexpr uint64_t kHeadLocked = uint64_t{1} << 7;
+
+    // Marker in a "with_shift" pointer for the end of a chain. Must also
+    // point back to the head of the chain (with end marker removed).
+    // Also includes the "locked" bit so that attempting to lock an empty
+    // chain has no effect (not needed, as the lock is only needed for
+    // removals).
+    static constexpr uint64_t kNextEndFlags = (uint64_t{1} << 6) | kHeadLocked;
+
+    static inline bool IsEnd(uint64_t next_with_shift) {
+      // Assuming certain values never used, suffices to check this one bit
+      constexpr auto kCheckBit = kNextEndFlags ^ kHeadLocked;
+      return next_with_shift & kCheckBit;
+    }
+
+    // Bottom bits to right shift away to get an array index from a
+    // "with_shift" pointer.
+    static constexpr int kNextShift = 8;
+
+    // A bit mask for the "shift" associated with each "with_shift" pointer.
+    // Always bottommost bits.
+    static constexpr int kShiftMask = 63;
+
+    // A marker for head_next_with_shift that indicates this HandleImpl is
+    // heap allocated (standalone) rather than in the table.
+    static constexpr uint64_t kStandaloneMarker = UINT64_MAX;
+
+    // A marker for head_next_with_shift indicating the head is not yet part
+    // of the usable table, or for chain_next_with_shift indicating that the
+    // entry is not present or is not yet part of a chain (must not be
+    // "shareable" state).
+    static constexpr uint64_t kUnusedMarker = 0;
+
+    // See above. The head pointer is logically independent of the rest of
+    // the entry, including the chain next pointer.
+    std::atomic<uint64_t> head_next_with_shift{kUnusedMarker};
+    std::atomic<uint64_t> chain_next_with_shift{kUnusedMarker};
+
+    // For supporting CreateStandalone and some fallback cases.
+    inline bool IsStandalone() const {
+      return head_next_with_shift.load(std::memory_order_acquire) ==
+             kStandaloneMarker;
+    }
+
+    inline void SetStandalone() {
+      head_next_with_shift.store(kStandaloneMarker, std::memory_order_release);
+    }
+  };  // struct HandleImpl
 
-  // A reference to Cache::eviction_callback_
-  const Cache::EvictionCallback& eviction_callback_;
+  struct Opts {
+    explicit Opts(size_t _min_avg_value_size)
+        : min_avg_value_size(_min_avg_value_size) {}
+
+    explicit Opts(const HyperClockCacheOptions& opts) {
+      assert(opts.estimated_entry_charge == 0);
+      min_avg_value_size = opts.min_avg_entry_charge;
+    }
+    size_t min_avg_value_size;
+  };
 
-  // A reference to ShardedCacheBase::hash_seed_
-  const uint32_t& hash_seed_;
+  AutoHyperClockTable(size_t capacity, bool strict_capacity_limit,
+                      CacheMetadataChargePolicy metadata_charge_policy,
+                      MemoryAllocator* allocator,
+                      const Cache::EvictionCallback* eviction_callback,
+                      const uint32_t* hash_seed, const Opts& opts);
+  ~AutoHyperClockTable();
+
+  // For BaseClockTable::Insert
+  struct InsertState {
+    uint64_t saved_length_info = 0;
+    size_t likely_empty_slot = 0;
+  };
 
-  // We partition the following members into different cache lines
-  // to avoid false sharing among Lookup, Release, Erase and Insert
-  // operations in ClockCacheShard.
+  void StartInsert(InsertState& state);
 
-  ALIGN_AS(CACHE_LINE_SIZE)
-  // Clock algorithm sweep pointer.
-  std::atomic<uint64_t> clock_pointer_{};
+  // Does initial check for whether there's hash table room for another
+  // inserted entry, possibly growing if needed. Returns true iff (after
+  // the call) there is room for the proposed number of entries.
+  bool GrowIfNeeded(size_t new_occupancy, InsertState& state);
 
-  ALIGN_AS(CACHE_LINE_SIZE)
-  // Number of elements in the table.
-  std::atomic<size_t> occupancy_{};
+  HandleImpl* DoInsert(const ClockHandleBasicData& proto,
+                       uint64_t initial_countdown, bool take_ref,
+                       InsertState& state);
 
-  // Memory usage by entries tracked by the cache (including standalone)
-  std::atomic<size_t> usage_{};
+  // Runs the clock eviction algorithm trying to reclaim at least
+  // requested_charge. Returns how much is evicted, which could be less
+  // if it appears impossible to evict the requested amount without blocking.
+  void Evict(size_t requested_charge, InsertState& state, EvictionData* data);
 
-  // Part of usage by standalone entries (not in table)
-  std::atomic<size_t> standalone_usage_{};
-};  // class HyperClockTable
+  HandleImpl* Lookup(const UniqueId64x2& hashed_key);
+
+  bool Release(HandleImpl* handle, bool useful, bool erase_if_last_ref);
+
+  void Erase(const UniqueId64x2& hashed_key);
+
+  void EraseUnRefEntries();
+
+  size_t GetTableSize() const;
+
+  size_t GetOccupancyLimit() const;
+
+  const HandleImpl* HandlePtr(size_t idx) const { return &array_[idx]; }
+
+#ifndef NDEBUG
+  size_t& TEST_MutableOccupancyLimit() {
+    return *reinterpret_cast<size_t*>(&occupancy_limit_);
+  }
+
+  // Release N references
+  void TEST_ReleaseN(HandleImpl* handle, size_t n);
+#endif
+
+  // Maximum ratio of number of occupied slots to number of usable slots. The
+  // actual load factor should float pretty close to this number, which should
+  // be a nice space/time trade-off, though large swings in WriteBufferManager
+  // memory could lead to low (but very much safe) load factors (only after
+  // seeing high load factors). Linear hashing along with (modified) linear
+  // probing to find an available slot increases potential risks of high
+  // load factors, so are disallowed.
+  static constexpr double kMaxLoadFactor = 0.60;
+
+ private:  // functions
+  // Returns true iff increased usable length. Due to load factor
+  // considerations, GrowIfNeeded might call this more than once to make room
+  // for one more entry.
+  bool Grow(InsertState& state);
+
+  // Operational details of splitting a chain into two for Grow().
+  void SplitForGrow(size_t grow_home, size_t old_home, int old_shift);
+
+  // Takes an "under construction" entry and ensures it is no longer connected
+  // to its home chain (in preparaion for completing erasure and freeing the
+  // slot). Note that previous operations might have already noticed it being
+  // "under (de)construction" and removed it from its chain.
+  void Remove(HandleImpl* h);
+
+  // Try to take ownership of an entry and erase+remove it from the table.
+  // Returns true if successful. Could fail if
+  // * There are other references to the entry
+  // * Some other thread has exclusive ownership or has freed it.
+  bool TryEraseHandle(HandleImpl* h, bool holding_ref, bool mark_invisible);
+
+  // Calculates the appropriate maximum table size, for creating the memory
+  // mapping.
+  static size_t CalcMaxUsableLength(
+      size_t capacity, size_t min_avg_value_size,
+      CacheMetadataChargePolicy metadata_charge_policy);
+
+  // Shared helper function that implements removing entries from a chain
+  // with proper handling to ensure all existing data is seen even in the
+  // presence of concurrent insertions, etc. (See implementation.)
+  template <class OpData>
+  void PurgeImpl(OpData* op_data, size_t home = SIZE_MAX);
+
+  // An RAII wrapper for locking a chain of entries for removals. See
+  // implementation.
+  class ChainRewriteLock;
+
+  // Helper function for PurgeImpl while holding a ChainRewriteLock. See
+  // implementation.
+  template <class OpData>
+  void PurgeImplLocked(OpData* op_data, ChainRewriteLock& rewrite_lock,
+                       size_t home);
+
+  // Update length_info_ as much as possible without waiting, given a known
+  // usable (ready for inserts and lookups) grow_home. (Previous grow_homes
+  // might not be usable yet, but we can check if they are by looking at
+  // the corresponding old home.)
+  void CatchUpLengthInfoNoWait(size_t known_usable_grow_home);
+
+ private:  // data
+  // mmaped area holding handles
+  const TypedMemMapping<HandleImpl> array_;
+
+  // Metadata for table size under linear hashing.
+  //
+  // Lowest 8 bits are the minimum number of lowest hash bits to use
+  // ("min shift"). The upper 56 bits are a threshold. If that minumum number
+  // of bits taken from a hash value is < this threshold, then one more bit of
+  // hash value is taken and used.
+  //
+  // Other mechanisms (shift amounts on pointers) ensure complete availability
+  // of data already in the table even if a reader only sees a completely
+  // out-of-date version of this value. In the worst case, it could take
+  // log time to find the correct chain, but normally this value enables
+  // readers to find the correct chain on the first try.
+  //
+  // NOTES: length_info_ is only updated at the end of a Grow operation,
+  // so that waiting in Grow operations isn't done while entries are pinned
+  // for internal operation purposes. Thus, Lookup and Insert have to
+  // detect and support cases where length_info hasn't caught up to updated
+  // chains. Winning grow thread is the one that transitions
+  // head_next_with_shift from zeros. Grow threads can spin/yield wait for
+  // preconditions and postconditions to be met.
+  std::atomic<uint64_t> length_info_;
+
+  // An already-computed version of the usable length times the max load
+  // factor. Could be slightly out of date but GrowIfNeeded()/Grow() handle
+  // that internally.
+  std::atomic<size_t> occupancy_limit_;
+
+  // See explanation in AutoHyperClockTable::Evict
+  std::atomic<size_t> clock_pointer_mask_;
+};  // class AutoHyperClockTable
 
 // A single shard of sharded cache.
-template <class Table>
+template <class TableT>
 class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase {
  public:
+  using Table = TableT;
   ClockCacheShard(size_t capacity, bool strict_capacity_limit,
                   CacheMetadataChargePolicy metadata_charge_policy,
                   MemoryAllocator* allocator,
@@ -666,9 +1054,17 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase {
     return Lookup(key, hashed_key);
   }
 
+  Table& GetTable() { return table_; }
+  const Table& GetTable() const { return table_; }
+
+#ifndef NDEBUG
+  size_t& TEST_MutableOccupancyLimit() {
+    return table_.TEST_MutableOccupancyLimit();
+  }
   // Acquire/release N references
   void TEST_RefN(HandleImpl* handle, size_t n);
   void TEST_ReleaseN(HandleImpl* handle, size_t n);
+#endif
 
  private:  // data
   Table table_;
@@ -680,17 +1076,14 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase {
   std::atomic<bool> strict_capacity_limit_;
 };  // class ClockCacheShard
 
-class HyperClockCache
-#ifdef NDEBUG
-    final
-#endif
-    : public ShardedCache<ClockCacheShard<HyperClockTable>> {
+template <class Table>
+class BaseHyperClockCache : public ShardedCache<ClockCacheShard<Table>> {
  public:
-  using Shard = ClockCacheShard<HyperClockTable>;
-
-  explicit HyperClockCache(const HyperClockCacheOptions& opts);
+  using Shard = ClockCacheShard<Table>;
+  using Handle = Cache::Handle;
+  using CacheItemHelper = Cache::CacheItemHelper;
 
-  const char* Name() const override { return "HyperClockCache"; }
+  explicit BaseHyperClockCache(const HyperClockCacheOptions& opts);
 
   Cache::ObjectPtr Value(Handle* handle) override;
 
@@ -700,7 +1093,35 @@ class HyperClockCache
 
   void ReportProblems(
       const std::shared_ptr<Logger>& /*info_log*/) const override;
-};  // class HyperClockCache
+};
+
+class FixedHyperClockCache
+#ifdef NDEBUG
+    final
+#endif
+    : public BaseHyperClockCache<FixedHyperClockTable> {
+ public:
+  using BaseHyperClockCache::BaseHyperClockCache;
+
+  const char* Name() const override { return "FixedHyperClockCache"; }
+
+  void ReportProblems(
+      const std::shared_ptr<Logger>& /*info_log*/) const override;
+};  // class FixedHyperClockCache
+
+class AutoHyperClockCache
+#ifdef NDEBUG
+    final
+#endif
+    : public BaseHyperClockCache<AutoHyperClockTable> {
+ public:
+  using BaseHyperClockCache::BaseHyperClockCache;
+
+  const char* Name() const override { return "AutoHyperClockCache"; }
+
+  void ReportProblems(
+      const std::shared_ptr<Logger>& /*info_log*/) const override;
+};  // class AutoHyperClockCache
 
 }  // namespace clock_cache