@nxtedition/rocksdb 15.4.0 → 15.5.0

package/deps/rocksdb/rocksdb/cache/clock_cache.cc

@@ -38,13 +38,11 @@ namespace ROCKSDB_NAMESPACE {
 namespace clock_cache {
 
 namespace {
-inline uint64_t GetRefcount(uint64_t meta) {
-  return ((meta >> ClockHandle::kAcquireCounterShift) -
-          (meta >> ClockHandle::kReleaseCounterShift)) &
-         ClockHandle::kCounterMask;
-}
+using SlotMeta = ClockHandle::SlotMeta;
+using AcquireCounter = SlotMeta::AcquireCounter;
+using ReleaseCounter = SlotMeta::ReleaseCounter;
 
-inline uint64_t GetInitialCountdown(Cache::Priority priority) {
+inline uint32_t GetInitialCountdown(Cache::Priority priority) {
   // Set initial clock data from priority
   // TODO: configuration parameters for priority handling and clock cycle
   // count?
@@ -65,11 +63,11 @@ inline uint64_t GetInitialCountdown(Cache::Priority priority) {
 inline void MarkEmpty(ClockHandle& h) {
 #ifndef NDEBUG
   // Mark slot as empty, with assertion
-  uint64_t meta = h.meta.Exchange(0);
-  assert(meta >> ClockHandle::kStateShift == ClockHandle::kStateConstruction);
+  auto old_meta = h.meta.Exchange({});
+  assert(old_meta.IsUnderConstruction());
 #else
   // Mark slot as empty
-  h.meta.Store(0);
+  h.meta.Store({});
 #endif
 }
 
@@ -85,18 +83,20 @@ inline void FreeDataMarkEmpty(ClockHandle& h, MemoryAllocator* allocator) {
 
 // Called to undo the effect of referencing an entry for internal purposes,
 // so it should not be marked as having been used.
-inline void Unref(const ClockHandle& h, uint64_t count = 1) {
+inline void Unref(const ClockHandle& h, uint32_t count = 1) {
   // Pretend we never took the reference
   // WART: there's a tiny chance we release last ref to invisible
   // entry here. If that happens, we let eviction take care of it.
-  uint64_t old_meta = h.meta.FetchSub(ClockHandle::kAcquireIncrement * count);
-  assert(GetRefcount(old_meta) != 0);
+  SlotMeta old_meta;
+  h.meta.Apply(AcquireCounter::MinusTransformPromiseNoUnderflow(count),
+               &old_meta);
+  assert(old_meta.GetRefcount() != 0);
   (void)old_meta;
 }
 
 inline bool ClockUpdate(ClockHandle& h, BaseClockTable::EvictionData* data,
                         bool* purgeable = nullptr) {
-  uint64_t meta;
+  SlotMeta meta;
   if (purgeable) {
     assert(*purgeable == false);
     // In AutoHCC, our eviction process follows the chain structure, so we
@@ -110,46 +110,40 @@ inline bool ClockUpdate(ClockHandle& h, BaseClockTable::EvictionData* data,
     meta = h.meta.LoadRelaxed();
   }
 
-  if (((meta >> ClockHandle::kStateShift) & ClockHandle::kStateShareableBit) ==
-      0) {
+  if (!meta.IsShareable()) {
     // Only clock update Shareable entries
     if (purgeable) {
       *purgeable = true;
       // AutoHCC only: make sure we only attempt to update non-empty slots
-      assert((meta >> ClockHandle::kStateShift) &
-             ClockHandle::kStateOccupiedBit);
+      assert(!meta.IsEmpty());
     }
     return false;
   }
-  uint64_t acquire_count =
-      (meta >> ClockHandle::kAcquireCounterShift) & ClockHandle::kCounterMask;
-  uint64_t release_count =
-      (meta >> ClockHandle::kReleaseCounterShift) & ClockHandle::kCounterMask;
+  uint32_t acquire_count = meta.GetAcquireCounter();
+  uint32_t release_count = meta.GetReleaseCounter();
   if (acquire_count != release_count) {
     // Only clock update entries with no outstanding refs
     data->seen_pinned_count++;
     return false;
   }
-  if ((meta >> ClockHandle::kStateShift == ClockHandle::kStateVisible) &&
-      acquire_count > 0) {
+  if (meta.IsVisible() && acquire_count > 0) {
     // Decrement clock
-    uint64_t new_count =
-        std::min(acquire_count - 1, uint64_t{ClockHandle::kMaxCountdown} - 1);
+    uint32_t new_count =
+        std::min(acquire_count - 1, uint32_t{ClockHandle::kMaxCountdown} - 1);
     // Compare-exchange in the decremented clock info, but
     // not aggressively
-    uint64_t new_meta =
-        (uint64_t{ClockHandle::kStateVisible} << ClockHandle::kStateShift) |
-        (meta & ClockHandle::kHitBitMask) |
-        (new_count << ClockHandle::kReleaseCounterShift) |
-        (new_count << ClockHandle::kAcquireCounterShift);
+    SlotMeta new_meta = meta;
+    new_meta.SetReleaseCounter(new_count);
+    new_meta.SetAcquireCounter(new_count);
     h.meta.CasStrongRelaxed(meta, new_meta);
     return false;
   }
   // Otherwise, remove entry (either unreferenced invisible or
   // unreferenced and expired visible).
-  if (h.meta.CasStrong(meta, (uint64_t{ClockHandle::kStateConstruction}
-                              << ClockHandle::kStateShift) |
-                                 (meta & ClockHandle::kHitBitMask))) {
+  SlotMeta construction_meta;
+  construction_meta.SetUnderConstruction();
+  construction_meta.SetHit(meta.GetHit());
+  if (h.meta.CasStrong(meta, construction_meta)) {
     // Took ownership.
     data->freed_charge += h.GetTotalCharge();
     data->freed_count += 1;
@@ -215,39 +209,39 @@ inline bool ClockUpdate(ClockHandle& h, BaseClockTable::EvictionData* data,
 // counter to reach "high" state again and bumped back to "medium." (This
 // motivates only checking for release counter in high state, not both in high
 // state.)
-inline void CorrectNearOverflow(uint64_t old_meta,
-                                AcqRelAtomic<uint64_t>& meta) {
+inline void CorrectNearOverflow(SlotMeta old_meta,
+                                BitFieldsAtomic<SlotMeta>& meta) {
   // We clear both top-most counter bits at the same time.
-  constexpr uint64_t kCounterTopBit = uint64_t{1}
-                                      << (ClockHandle::kCounterNumBits - 1);
-  constexpr uint64_t kClearBits =
-      (kCounterTopBit << ClockHandle::kAcquireCounterShift) |
-      (kCounterTopBit << ClockHandle::kReleaseCounterShift);
-  // A simple check that allows us to initiate clearing the top bits for
-  // a large portion of the "high" state space on release counter.
-  constexpr uint64_t kCheckBits =
-      (kCounterTopBit | (ClockHandle::kMaxCountdown + 1))
-      << ClockHandle::kReleaseCounterShift;
+  constexpr uint32_t kCounterTopBit = uint32_t{1}
+                                      << (SlotMeta::kCounterNumBits - 1);
+  // The threshold for correcting "near overflow" is to ensure
+  // (a) the value has a top bit set that can be cleared
+  // (b) when we clear the top bit, the eviction state will be preserved
+  //     (everything >= kMaxCountdown is treated equivalently)
+  // As mentioned above, we only check the release count.
+  constexpr uint32_t kThreshold = kCounterTopBit + ClockHandle::kMaxCountdown;
 
-  if (UNLIKELY(old_meta & kCheckBits)) {
-    meta.FetchAndRelaxed(~kClearBits);
+  if (UNLIKELY(old_meta.GetReleaseCounter() > kThreshold)) {
+    auto clear_transform = AcquireCounter::AndTransform(kCounterTopBit - 1) +
+                           ReleaseCounter::AndTransform(kCounterTopBit - 1);
+    meta.ApplyRelaxed(clear_transform);
   }
 }
 
 inline bool BeginSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h,
-                            uint64_t initial_countdown, bool* already_matches) {
+                            uint32_t initial_countdown, bool* already_matches) {
   assert(*already_matches == false);
   // Optimistically transition the slot from "empty" to
   // "under construction" (no effect on other states)
-  uint64_t old_meta = h.meta.FetchOr(uint64_t{ClockHandle::kStateOccupiedBit}
-                                     << ClockHandle::kStateShift);
-  uint64_t old_state = old_meta >> ClockHandle::kStateShift;
+  auto set_occupied = SlotMeta::OccupiedFlag::SetTransform();
+  SlotMeta old_meta;
+  h.meta.Apply(set_occupied, &old_meta);
 
-  if (old_state == ClockHandle::kStateEmpty) {
+  if (old_meta.IsEmpty()) {
     // We've started inserting into an available slot, and taken
     // ownership.
     return true;
-  } else if (old_state != ClockHandle::kStateVisible) {
+  } else if (!old_meta.IsVisible()) {
     // Slot not usable / touchable now
     return false;
   }
@@ -255,15 +249,17 @@ inline bool BeginSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h,
   // But first, we need to acquire a ref to read it. In fact, number of
   // refs for initial countdown, so that we boost the clock state if
   // this is a match.
-  old_meta =
-      h.meta.FetchAdd(ClockHandle::kAcquireIncrement * initial_countdown);
+  auto add_acquire =
+      AcquireCounter::PlusTransformPromiseNoOverflow(initial_countdown);
+  h.meta.Apply(add_acquire, &old_meta);
   // Like Lookup
-  if ((old_meta >> ClockHandle::kStateShift) == ClockHandle::kStateVisible) {
+  if (old_meta.IsVisible()) {
     // Acquired a read reference
     if (h.hashed_key == proto.hashed_key) {
       // Match. Release in a way that boosts the clock state
-      old_meta =
-          h.meta.FetchAdd(ClockHandle::kReleaseIncrement * initial_countdown);
+      auto add_release =
+          ReleaseCounter::PlusTransformPromiseNoOverflow(initial_countdown);
+      h.meta.Apply(add_release, &old_meta);
       // Correct for possible (but rare) overflow
       CorrectNearOverflow(old_meta, h.meta);
       // Insert detached instead (only if return handle needed)
@@ -273,8 +269,7 @@ inline bool BeginSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h,
       // Mismatch.
       Unref(h, initial_countdown);
     }
-  } else if (UNLIKELY((old_meta >> ClockHandle::kStateShift) ==
-                      ClockHandle::kStateInvisible)) {
+  } else if (UNLIKELY(old_meta.IsInvisible())) {
     // Pretend we never took the reference
     Unref(h, initial_countdown);
   } else {
@@ -286,25 +281,23 @@ inline bool BeginSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h,
 }
 
 inline void FinishSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h,
-                             uint64_t initial_countdown, bool keep_ref) {
+                             uint32_t initial_countdown, bool keep_ref) {
   // Save data fields
   ClockHandleBasicData* h_alias = &h;
   *h_alias = proto;
 
   // Transition from "under construction" state to "visible" state
-  uint64_t new_meta = uint64_t{ClockHandle::kStateVisible}
-                      << ClockHandle::kStateShift;
+  SlotMeta new_meta;
+  new_meta.SetVisible();
 
   // Maybe with an outstanding reference
-  new_meta |= initial_countdown << ClockHandle::kAcquireCounterShift;
-  new_meta |= (initial_countdown - keep_ref)
-              << ClockHandle::kReleaseCounterShift;
+  new_meta.SetAcquireCounter(initial_countdown);
+  new_meta.SetReleaseCounter(initial_countdown - (keep_ref ? 1 : 0));
 
 #ifndef NDEBUG
   // Save the state transition, with assertion
-  uint64_t old_meta = h.meta.Exchange(new_meta);
-  assert(old_meta >> ClockHandle::kStateShift ==
-         ClockHandle::kStateConstruction);
+  auto old_meta = h.meta.Exchange(new_meta);
+  assert(old_meta.IsUnderConstruction());
 #else
   // Save the state transition
   h.meta.Store(new_meta);
@@ -312,7 +305,7 @@ inline void FinishSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h,
 }
 
 bool TryInsert(const ClockHandleBasicData& proto, ClockHandle& h,
-               uint64_t initial_countdown, bool keep_ref,
+               uint32_t initial_countdown, bool keep_ref,
                bool* already_matches) {
   bool b = BeginSlotInsert(proto, h, initial_countdown, already_matches);
   if (b) {
@@ -326,35 +319,32 @@ template <class HandleImpl, class Func>
 void ConstApplyToEntriesRange(const Func& func, const HandleImpl* begin,
                               const HandleImpl* end,
                               bool apply_if_will_be_deleted) {
-  uint64_t check_state_mask = ClockHandle::kStateShareableBit;
-  if (!apply_if_will_be_deleted) {
-    check_state_mask |= ClockHandle::kStateVisibleBit;
-  }
-
   for (const HandleImpl* h = begin; h < end; ++h) {
     // Note: to avoid using compare_exchange, we have to be extra careful.
-    uint64_t old_meta = h->meta.LoadRelaxed();
+    SlotMeta old_meta = h->meta.LoadRelaxed();
     // Check if it's an entry visible to lookups
-    if ((old_meta >> ClockHandle::kStateShift) & check_state_mask) {
-      // Increment acquire counter. Note: it's possible that the entry has
-      // completely changed since we loaded old_meta, but incrementing acquire
-      // count is always safe. (Similar to optimistic Lookup here.)
-      old_meta = h->meta.FetchAdd(ClockHandle::kAcquireIncrement);
-      // Check whether we actually acquired a reference.
-      if ((old_meta >> ClockHandle::kStateShift) &
-          ClockHandle::kStateShareableBit) {
-        // Apply func if appropriate
-        if ((old_meta >> ClockHandle::kStateShift) & check_state_mask) {
-          func(*h);
+    if (apply_if_will_be_deleted || old_meta.IsVisible()) {
+      if (old_meta.IsShareable()) {
+        // Increment acquire counter. Note: it's possible that the entry has
+        // completely changed since we loaded old_meta, but incrementing acquire
+        // count is always safe. (Similar to optimistic Lookup here.)
+        auto add_acquire = AcquireCounter::PlusTransformPromiseNoOverflow(1);
+        h->meta.Apply(add_acquire, &old_meta);
+        // Check whether we actually acquired a reference.
+        if (old_meta.IsShareable()) {
+          // Apply func if appropriate
+          if (apply_if_will_be_deleted || old_meta.IsVisible()) {
+            func(*h);
+          }
+          // Pretend we never took the reference
+          Unref(*h);
+          // No net change, so don't need to check for overflow
+        } else {
+          // For other states, incrementing the acquire counter has no effect
+          // so we don't need to undo it. Furthermore, we cannot safely undo
+          // it because we did not acquire a read reference to lock the
+          // entry in a Shareable state.
         }
-        // Pretend we never took the reference
-        Unref(*h);
-        // No net change, so don't need to check for overflow
-      } else {
-        // For other states, incrementing the acquire counter has no effect
-        // so we don't need to undo it. Furthermore, we cannot safely undo
-        // it because we did not acquire a read reference to lock the
-        // entry in a Shareable state.
       }
     }
   }
@@ -399,9 +389,9 @@ HandleImpl* BaseClockTable::StandaloneInsert(
   h->SetStandalone();
   // Single reference (standalone entries only created if returning a refed
   // Handle back to user)
-  uint64_t meta = uint64_t{ClockHandle::kStateInvisible}
-                  << ClockHandle::kStateShift;
-  meta |= uint64_t{1} << ClockHandle::kAcquireCounterShift;
+  SlotMeta meta;
+  meta.SetInvisible();
+  meta.SetAcquireCounter(1);
   h->meta.Store(meta);
   // Keep track of how much of usage is standalone
   standalone_usage_.FetchAddRelaxed(proto.GetTotalCharge());
@@ -564,11 +554,10 @@ void BaseClockTable::TrackAndReleaseEvictedEntry(ClockHandle* h) {
   if (eviction_callback_) {
     // For key reconstructed from hash
     UniqueId64x2 unhashed;
-    took_value_ownership =
-        eviction_callback_(ClockCacheShard<FixedHyperClockTable>::ReverseHash(
-                               h->GetHash(), &unhashed, hash_seed_),
-                           static_cast<Cache::Handle*>(h),
-                           h->meta.LoadRelaxed() & ClockHandle::kHitBitMask);
+    took_value_ownership = eviction_callback_(
+        ClockCacheShard<FixedHyperClockTable>::ReverseHash(
+            h->GetHash(), &unhashed, hash_seed_),
+        static_cast<Cache::Handle*>(h), h->meta.LoadRelaxed().GetHit());
   }
   if (!took_value_ownership) {
     h->FreeData(allocator_);
@@ -648,7 +637,7 @@ Status BaseClockTable::Insert(const ClockHandleBasicData& proto,
     // * Have to insert into a suboptimal location (more probes) so that the
     // old entry can be kept around as well.
 
-    uint64_t initial_countdown = GetInitialCountdown(priority);
+    uint32_t initial_countdown = GetInitialCountdown(priority);
     assert(initial_countdown > 0);
 
     HandleImpl* e =
@@ -693,34 +682,34 @@ Status BaseClockTable::Insert(const ClockHandleBasicData& proto,
 
 void BaseClockTable::Ref(ClockHandle& h) {
   // Increment acquire counter
-  uint64_t old_meta = h.meta.FetchAdd(ClockHandle::kAcquireIncrement);
+  SlotMeta old_meta;
+  h.meta.Apply(AcquireCounter::PlusTransformPromiseNoOverflow(1), &old_meta);
 
-  assert((old_meta >> ClockHandle::kStateShift) &
-         ClockHandle::kStateShareableBit);
+  assert(old_meta.IsShareable());
   // Must have already had a reference
-  assert(GetRefcount(old_meta) > 0);
+  assert(old_meta.GetRefcount() > 0);
   (void)old_meta;
 }
 
 #ifndef NDEBUG
-void BaseClockTable::TEST_RefN(ClockHandle& h, size_t n) {
+void BaseClockTable::TEST_RefN(ClockHandle& h, uint32_t n) {
   // Increment acquire counter
-  uint64_t old_meta = h.meta.FetchAdd(n * ClockHandle::kAcquireIncrement);
+  SlotMeta old_meta;
+  h.meta.Apply(AcquireCounter::PlusTransformPromiseNoOverflow(n), &old_meta);
 
-  assert((old_meta >> ClockHandle::kStateShift) &
-         ClockHandle::kStateShareableBit);
+  assert(old_meta.IsShareable());
   (void)old_meta;
 }
 
-void BaseClockTable::TEST_ReleaseNMinus1(ClockHandle* h, size_t n) {
+void BaseClockTable::TEST_ReleaseNMinus1(ClockHandle* h, uint32_t n) {
   assert(n > 0);
 
   // Like n-1 Releases, but assumes one more will happen in the caller to take
   // care of anything like erasing an unreferenced, invisible entry.
-  uint64_t old_meta =
-      h->meta.FetchAdd((n - 1) * ClockHandle::kReleaseIncrement);
-  assert((old_meta >> ClockHandle::kStateShift) &
-         ClockHandle::kStateShareableBit);
+  SlotMeta old_meta;
+  h->meta.Apply(ReleaseCounter::PlusTransformPromiseNoOverflow(n - 1),
+                &old_meta);
+  assert(old_meta.IsShareable());
   (void)old_meta;
 }
 #endif
@@ -754,23 +743,20 @@ FixedHyperClockTable::~FixedHyperClockTable() {
   // in the table.
   for (size_t i = 0; i < GetTableSize(); i++) {
     HandleImpl& h = array_[i];
-    switch (h.meta.LoadRelaxed() >> ClockHandle::kStateShift) {
-      case ClockHandle::kStateEmpty:
-        // noop
-        break;
-      case ClockHandle::kStateInvisible:  // rare but possible
-      case ClockHandle::kStateVisible:
-        assert(GetRefcount(h.meta.LoadRelaxed()) == 0);
-        h.FreeData(allocator_);
+    SlotMeta meta = h.meta.LoadRelaxed();
+    if (meta.IsShareable()) {
+      // NOTE: Reaching here invisible is rare but possible
+      assert(meta.GetRefcount() == 0);
+      h.FreeData(allocator_);
 #ifndef NDEBUG
-        Rollback(h.hashed_key, &h);
-        ReclaimEntryUsage(h.GetTotalCharge());
+      Rollback(h.hashed_key, &h);
+      ReclaimEntryUsage(h.GetTotalCharge());
 #endif
-        break;
-      // otherwise
-      default:
-        assert(false);
-        break;
+    } else {
+      // Should be no transient "under construction" states unless a thread
+      // was killed or we are being destructed while another thread is still
+      // operating on the structure
+      assert(meta.IsEmpty());
     }
   }
 
@@ -792,7 +778,7 @@ bool FixedHyperClockTable::GrowIfNeeded(size_t new_occupancy, InsertState&) {
 }
 
 FixedHyperClockTable::HandleImpl* FixedHyperClockTable::DoInsert(
-    const ClockHandleBasicData& proto, uint64_t initial_countdown,
+    const ClockHandleBasicData& proto, uint32_t initial_countdown,
     bool keep_ref, InsertState&) {
   bool already_matches = false;
   HandleImpl* e = FindSlot(
@@ -843,47 +829,46 @@ FixedHyperClockTable::HandleImpl* FixedHyperClockTable::Lookup(
   HandleImpl* e = FindSlot(
       hashed_key,
       [&](HandleImpl* h) {
+        SlotMeta old_meta;
         // Mostly branch-free version (similar performance)
         /*
-        uint64_t old_meta = h->meta.FetchAdd(ClockHandle::kAcquireIncrement,
-                                     std::memory_order_acquire);
-        bool Shareable = (old_meta >> (ClockHandle::kStateShift + 1)) & 1U;
-        bool visible = (old_meta >> ClockHandle::kStateShift) & 1U;
-        bool match = (h->key == key) & visible;
-        h->meta.FetchSub(static_cast<uint64_t>(Shareable & !match) <<
-        ClockHandle::kAcquireCounterShift); return
-        match;
+        h->meta.Apply(AcquireCounter::PlusTransformPromiseNoOverflow(1),
+                      &old_meta);
+        bool shareable = old_meta.IsShareable();
+        bool visible = old_meta.IsVisible();
+        bool match = (h->hashed_key == hashed_key) & visible;
+        h->meta.Apply(AcquireCounter::MinusTransformPromiseNoUnderflow(
+            uint32_t{shareable} & uint32_t{!match}));
+        h->meta.Apply(SlotMeta::HitFlag::Or(match));
+        return match;
         */
         // Optimistic lookup should pay off when the table is relatively
         // sparse.
         constexpr bool kOptimisticLookup = true;
-        uint64_t old_meta;
         if (!kOptimisticLookup) {
           old_meta = h->meta.Load();
-          if ((old_meta >> ClockHandle::kStateShift) !=
-              ClockHandle::kStateVisible) {
+          if (!old_meta.IsVisible()) {
             return false;
           }
         }
         // (Optimistically) increment acquire counter
-        old_meta = h->meta.FetchAdd(ClockHandle::kAcquireIncrement);
+        h->meta.Apply(AcquireCounter::PlusTransformPromiseNoOverflow(1),
+                      &old_meta);
         // Check if it's an entry visible to lookups
-        if ((old_meta >> ClockHandle::kStateShift) ==
-            ClockHandle::kStateVisible) {
+        if (old_meta.IsVisible()) {
           // Acquired a read reference
           if (h->hashed_key == hashed_key) {
             // Match
             // Update the hit bit
             if (eviction_callback_) {
-              h->meta.FetchOrRelaxed(uint64_t{1} << ClockHandle::kHitBitShift);
+              h->meta.ApplyRelaxed(SlotMeta::HitFlag::SetTransform());
             }
             return true;
           } else {
             // Mismatch. Pretend we never took the reference
             Unref(*h);
           }
-        } else if (UNLIKELY((old_meta >> ClockHandle::kStateShift) ==
-                            ClockHandle::kStateInvisible)) {
+        } else if (UNLIKELY(old_meta.IsInvisible())) {
           // Pretend we never took the reference
           Unref(*h);
         } else {
@@ -907,53 +892,49 @@ bool FixedHyperClockTable::Release(HandleImpl* h, bool useful,
   // is only freed up by EvictFromClock (called by Insert when space is needed)
   // and Erase. We do this to avoid an extra atomic read of the variable usage_.
 
-  uint64_t old_meta;
+  SlotMeta old_meta;
   if (useful) {
     // Increment release counter to indicate was used
-    old_meta = h->meta.FetchAdd(ClockHandle::kReleaseIncrement);
+    auto add_release = ReleaseCounter::PlusTransformPromiseNoOverflow(1);
+    h->meta.Apply(add_release, &old_meta);
   } else {
     // Decrement acquire counter to pretend it never happened
-    old_meta = h->meta.FetchSub(ClockHandle::kAcquireIncrement);
+    auto sub_acquire = AcquireCounter::MinusTransformPromiseNoUnderflow(1);
+    h->meta.Apply(sub_acquire, &old_meta);
   }
 
-  assert((old_meta >> ClockHandle::kStateShift) &
-         ClockHandle::kStateShareableBit);
+  assert(old_meta.IsShareable());
   // No underflow
-  assert(((old_meta >> ClockHandle::kAcquireCounterShift) &
-          ClockHandle::kCounterMask) !=
-         ((old_meta >> ClockHandle::kReleaseCounterShift) &
-          ClockHandle::kCounterMask));
+  assert(old_meta.GetAcquireCounter() != old_meta.GetReleaseCounter());
 
-  if (erase_if_last_ref || UNLIKELY(old_meta >> ClockHandle::kStateShift ==
-                                    ClockHandle::kStateInvisible)) {
+  if (erase_if_last_ref || UNLIKELY(old_meta.IsInvisible())) {
     // FIXME: There's a chance here that another thread could replace this
     // entry and we end up erasing the wrong one.
 
-    // Update for last FetchAdd op
+    // Update for last Apply op
     if (useful) {
-      old_meta += ClockHandle::kReleaseIncrement;
+      old_meta.SetReleaseCounter(old_meta.GetReleaseCounter() + 1);
     } else {
-      old_meta -= ClockHandle::kAcquireIncrement;
+      old_meta.SetAcquireCounter(old_meta.GetAcquireCounter() - 1);
     }
     // Take ownership if no refs
+    SlotMeta construction_meta;
+    construction_meta.SetUnderConstruction();
     do {
-      if (GetRefcount(old_meta) != 0) {
+      if (old_meta.GetRefcount() != 0) {
         // Not last ref at some point in time during this Release call
         // Correct for possible (but rare) overflow
         CorrectNearOverflow(old_meta, h->meta);
         return false;
       }
-      if ((old_meta & (uint64_t{ClockHandle::kStateShareableBit}
-                       << ClockHandle::kStateShift)) == 0) {
+      if (!old_meta.IsShareable()) {
         // Someone else took ownership
         return false;
       }
       // Note that there's a small chance that we release, another thread
       // replaces this entry with another, reaches zero refs, and then we end
       // up erasing that other entry. That's an acceptable risk / imprecision.
-    } while (
-        !h->meta.CasWeak(old_meta, uint64_t{ClockHandle::kStateConstruction}
-                                       << ClockHandle::kStateShift));
+    } while (!h->meta.CasWeak(old_meta, construction_meta));
     // Took ownership
     size_t total_charge = h->GetTotalCharge();
     if (UNLIKELY(h->IsStandalone())) {
@@ -976,7 +957,7 @@ bool FixedHyperClockTable::Release(HandleImpl* h, bool useful,
 }
 
 #ifndef NDEBUG
-void FixedHyperClockTable::TEST_ReleaseN(HandleImpl* h, size_t n) {
+void FixedHyperClockTable::TEST_ReleaseN(HandleImpl* h, uint32_t n) {
   if (n > 0) {
     // Do n-1 simple releases first
     TEST_ReleaseNMinus1(h, n);
@@ -993,30 +974,29 @@ void FixedHyperClockTable::Erase(const UniqueId64x2& hashed_key) {
       [&](HandleImpl* h) {
         // Could be multiple entries in rare cases. Erase them all.
         // Optimistically increment acquire counter
-        uint64_t old_meta = h->meta.FetchAdd(ClockHandle::kAcquireIncrement);
+        auto add_acquire = AcquireCounter::PlusTransformPromiseNoOverflow(1);
+        SlotMeta old_meta, meta;
+        h->meta.Apply(add_acquire, &old_meta, &meta);
         // Check if it's an entry visible to lookups
-        if ((old_meta >> ClockHandle::kStateShift) ==
-            ClockHandle::kStateVisible) {
+        if (meta.IsVisible()) {
           // Acquired a read reference
           if (h->hashed_key == hashed_key) {
-            // Match. Set invisible.
-            old_meta =
-                h->meta.FetchAnd(~(uint64_t{ClockHandle::kStateVisibleBit}
-                                   << ClockHandle::kStateShift));
-            // Apply update to local copy
-            old_meta &= ~(uint64_t{ClockHandle::kStateVisibleBit}
-                          << ClockHandle::kStateShift);
+            // Match. Take ownership if no other refs, or set invisible other
+            // refs exist.
             for (;;) {
-              uint64_t refcount = GetRefcount(old_meta);
+              uint32_t refcount = meta.GetRefcount();
               assert(refcount > 0);
               if (refcount > 1) {
                 // Not last ref at some point in time during this Erase call
-                // Pretend we never took the reference
+                // Set invisible
+                h->meta.Apply(SlotMeta::VisibleFlag::ClearTransform());
+                // And pretend we never took the reference
                 Unref(*h);
                 break;
-              } else if (h->meta.CasWeak(
-                             old_meta, uint64_t{ClockHandle::kStateConstruction}
-                                           << ClockHandle::kStateShift)) {
+              }
+              SlotMeta construction_meta;
+              construction_meta.SetUnderConstruction();
+              if (h->meta.CasWeak(meta, construction_meta)) {
                 // Took ownership
                 assert(hashed_key == h->hashed_key);
                 size_t total_charge = h->GetTotalCharge();
@@ -1032,8 +1012,7 @@ void FixedHyperClockTable::Erase(const UniqueId64x2& hashed_key) {
             // Mismatch. Pretend we never took the reference
             Unref(*h);
           }
-        } else if (UNLIKELY((old_meta >> ClockHandle::kStateShift) ==
-                            ClockHandle::kStateInvisible)) {
+        } else if (UNLIKELY(old_meta.IsInvisible())) {
           // Pretend we never took the reference
           Unref(*h);
         } else {
@@ -1050,17 +1029,17 @@ void FixedHyperClockTable::EraseUnRefEntries() {
   for (size_t i = 0; i <= this->length_bits_mask_; i++) {
     HandleImpl& h = array_[i];
 
-    uint64_t old_meta = h.meta.LoadRelaxed();
-    if (old_meta & (uint64_t{ClockHandle::kStateShareableBit}
-                    << ClockHandle::kStateShift) &&
-        GetRefcount(old_meta) == 0 &&
-        h.meta.CasStrong(old_meta, uint64_t{ClockHandle::kStateConstruction}
-                                       << ClockHandle::kStateShift)) {
-      // Took ownership
-      size_t total_charge = h.GetTotalCharge();
-      Rollback(h.hashed_key, &h);
-      FreeDataMarkEmpty(h, allocator_);
-      ReclaimEntryUsage(total_charge);
+    SlotMeta old_meta = h.meta.LoadRelaxed();
+    if (old_meta.IsShareable() && old_meta.GetRefcount() == 0) {
+      SlotMeta construction_meta;
+      construction_meta.SetUnderConstruction();
+      if (h.meta.CasStrong(old_meta, construction_meta)) {
+        // Took ownership
+        size_t total_charge = h.GetTotalCharge();
+        Rollback(h.hashed_key, &h);
+        FreeDataMarkEmpty(h, allocator_);
+        ReclaimEntryUsage(total_charge);
+      }
     }
   }
 }
@@ -1320,12 +1299,12 @@ bool ClockCacheShard<Table>::Release(HandleImpl* handle, bool useful,
 
 #ifndef NDEBUG
 template <class Table>
-void ClockCacheShard<Table>::TEST_RefN(HandleImpl* h, size_t n) {
+void ClockCacheShard<Table>::TEST_RefN(HandleImpl* h, uint32_t n) {
   table_.TEST_RefN(*h, n);
 }
 
 template <class Table>
-void ClockCacheShard<Table>::TEST_ReleaseN(HandleImpl* h, size_t n) {
+void ClockCacheShard<Table>::TEST_ReleaseN(HandleImpl* h, uint32_t n) {
   table_.TEST_ReleaseN(h, n);
 }
 #endif
@@ -1373,8 +1352,8 @@ size_t ClockCacheShard<Table>::GetPinnedUsage() const {
       metadata_charge_policy_ == kFullChargeCacheMetadata;
   ConstApplyToEntriesRange(
       [&table_pinned_usage, charge_metadata](const HandleImpl& h) {
-        uint64_t meta = h.meta.LoadRelaxed();
-        uint64_t refcount = GetRefcount(meta);
+        SlotMeta meta = h.meta.LoadRelaxed();
+        uint32_t refcount = meta.GetRefcount();
         // Holding one ref for ConstApplyToEntriesRange
         assert(refcount > 0);
         if (refcount > 1) {
@@ -1494,7 +1473,7 @@ void AddShardEvaluation(const FixedHyperClockCache::Shard& shard,
 }
 
 bool IsSlotOccupied(const ClockHandle& h) {
-  return (h.meta.LoadRelaxed() >> ClockHandle::kStateShift) != 0;
+  return !h.meta.LoadRelaxed().IsEmpty();
 }
 }  // namespace
 
@@ -1759,12 +1738,12 @@ inline bool MatchAndRef(const UniqueId64x2* hashed_key, const ClockHandle& h,
   // Must be at least something to match
   assert(hashed_key || shift > 0);
 
-  uint64_t old_meta;
+  SlotMeta old_meta, new_meta;
   // (Optimistically) increment acquire counter.
-  old_meta = h.meta.FetchAdd(ClockHandle::kAcquireIncrement);
+  auto add_acquire = AcquireCounter::PlusTransformPromiseNoOverflow(1);
+  h.meta.Apply(add_acquire, &old_meta, &new_meta);
   // Check if it's a referencable (sharable) entry
-  if ((old_meta & (uint64_t{ClockHandle::kStateShareableBit}
-                   << ClockHandle::kStateShift)) == 0) {
+  if (!old_meta.IsShareable()) {
     // For non-sharable states, incrementing the acquire counter has no effect
     // so we don't need to undo it. Furthermore, we cannot safely undo
     // it because we did not acquire a read reference to lock the
@@ -1775,10 +1754,9 @@ inline bool MatchAndRef(const UniqueId64x2* hashed_key, const ClockHandle& h,
     return false;
   }
   // Else acquired a read reference
-  assert(GetRefcount(old_meta + ClockHandle::kAcquireIncrement) > 0);
+  assert(new_meta.GetRefcount() > 0);
   if (hashed_key && h.hashed_key == *hashed_key &&
-      LIKELY(old_meta & (uint64_t{ClockHandle::kStateVisibleBit}
-                         << ClockHandle::kStateShift))) {
+      LIKELY(old_meta.IsVisible())) {
     // Match on full key, visible
     if (full_match_or_unknown) {
       *full_match_or_unknown = true;
@@ -1946,7 +1924,7 @@ class AutoHyperClockTable::ChainRewriteLock {
     }
   }
 
-  AcqRelBitFieldsAtomic<NextWithShift>* head_ptr_;
+  BitFieldsAtomic<NextWithShift>* head_ptr_;
   NextWithShift saved_head_;
 };
 
@@ -2051,7 +2029,7 @@ AutoHyperClockTable::~AutoHyperClockTable() {
            HandleImpl::kUnusedMarker);
     assert(array_[i].chain_next_with_shift.LoadRelaxed() ==
            HandleImpl::kUnusedMarker);
-    assert(array_[i].meta.LoadRelaxed() == 0);
+    assert(array_[i].meta.LoadRelaxed() == SlotMeta{});
   }
 #endif          // MUST_FREE_HEAP_ALLOCATIONS
 #ifndef NDEBUG  // Extra invariant checking
@@ -2060,30 +2038,27 @@ AutoHyperClockTable::~AutoHyperClockTable() {
 #endif  // !NDEBUG
   for (size_t i = 0; i < used_end; i++) {
     HandleImpl& h = array_[i];
-    switch (h.meta.LoadRelaxed() >> ClockHandle::kStateShift) {
-      case ClockHandle::kStateEmpty:
-        // noop
-        break;
-      case ClockHandle::kStateInvisible:  // rare but possible
-      case ClockHandle::kStateVisible:
-        assert(GetRefcount(h.meta.LoadRelaxed()) == 0);
-        h.FreeData(allocator_);
+    SlotMeta meta = h.meta.LoadRelaxed();
+    if (meta.IsShareable()) {
+      // NOTE: Reaching here invisible is rare but possible
+      assert(meta.GetRefcount() == 0);
+      h.FreeData(allocator_);
 #ifndef NDEBUG  // Extra invariant checking
-        usage_.FetchSubRelaxed(h.total_charge);
-        occupancy_.FetchSubRelaxed(1U);
-        was_populated[i] = true;
-        if (!h.chain_next_with_shift.LoadRelaxed().IsEnd()) {
-          assert(!h.chain_next_with_shift.LoadRelaxed().IsLocked());
-          size_t next = h.chain_next_with_shift.LoadRelaxed().GetNext();
-          assert(!was_pointed_to[next]);
-          was_pointed_to[next] = true;
-        }
+      usage_.FetchSubRelaxed(h.total_charge);
+      occupancy_.FetchSubRelaxed(1U);
+      was_populated[i] = true;
+      if (!h.chain_next_with_shift.LoadRelaxed().IsEnd()) {
+        assert(!h.chain_next_with_shift.LoadRelaxed().IsLocked());
+        size_t next = h.chain_next_with_shift.LoadRelaxed().GetNext();
+        assert(!was_pointed_to[next]);
+        was_pointed_to[next] = true;
+      }
 #endif  // !NDEBUG
-        break;
-      // otherwise
-      default:
-        assert(false);
-        break;
+    } else {
+      // Should be no transient "under construction" states unless a thread
+      // was killed or we are being destructed while another thread is still
+      // operating on the structure
+      assert(meta.IsEmpty());
     }
 #ifndef NDEBUG  // Extra invariant checking
     if (!h.head_next_with_shift.LoadRelaxed().IsEnd()) {
@@ -2691,20 +2666,17 @@ void AutoHyperClockTable::PurgeImplLocked(OpData* op_data,
           op_data->push_back(h);
           // Entries for eviction become purgeable
           purgeable = true;
-          assert((h->meta.Load() >> ClockHandle::kStateShift) ==
-                 ClockHandle::kStateConstruction);
+          assert(h->meta.Load().IsUnderConstruction());
         }
       } else {
         (void)op_data;
         (void)data;
-        purgeable = ((h->meta.Load() >> ClockHandle::kStateShift) &
-                     ClockHandle::kStateShareableBit) == 0;
+        purgeable = !h->meta.Load().IsShareable();
       }
     }
 
     if (purgeable) {
-      assert((h->meta.Load() >> ClockHandle::kStateShift) ==
-             ClockHandle::kStateConstruction);
+      assert(h->meta.Load().IsUnderConstruction());
       pending_purge = true;
     } else if (pending_purge) {
       if (prev_to_keep) {
@@ -2864,7 +2836,7 @@ void AutoHyperClockTable::PurgeImpl(OpData* op_data, size_t home,
 }
 
 AutoHyperClockTable::HandleImpl* AutoHyperClockTable::DoInsert(
-    const ClockHandleBasicData& proto, uint64_t initial_countdown,
+    const ClockHandleBasicData& proto, uint32_t initial_countdown,
     bool take_ref, InsertState& state) {
   size_t home;
   int orig_home_shift;
@@ -3149,14 +3121,14 @@ AutoHyperClockTable::HandleImpl* AutoHyperClockTable::Lookup(
 #endif
     if (probably_equal) {
       // Increment acquire counter for definitive check
-      uint64_t old_meta = h->meta.FetchAdd(ClockHandle::kAcquireIncrement);
+      auto add_acquire = AcquireCounter::PlusTransformPromiseNoOverflow(1);
+      SlotMeta old_meta, new_meta;
+      h->meta.Apply(add_acquire, &old_meta, &new_meta);
       // Check if it's a referencable (sharable) entry
-      if (LIKELY(old_meta & (uint64_t{ClockHandle::kStateShareableBit}
-                             << ClockHandle::kStateShift))) {
-        assert(GetRefcount(old_meta + ClockHandle::kAcquireIncrement) > 0);
+      if (LIKELY(old_meta.IsShareable())) {
+        assert(new_meta.GetRefcount() > 0);
         if (LIKELY(h->hashed_key == hashed_key) &&
-            LIKELY(old_meta & (uint64_t{ClockHandle::kStateVisibleBit}
-                               << ClockHandle::kStateShift))) {
+            LIKELY(old_meta.IsVisible())) {
           return h;
         } else {
           Unref(*h);
@@ -3277,7 +3249,7 @@ AutoHyperClockTable::HandleImpl* AutoHyperClockTable::Lookup(
         }
         // Update the hit bit
         if (eviction_callback_) {
-          h->meta.FetchOrRelaxed(uint64_t{1} << ClockHandle::kHitBitShift);
+          h->meta.ApplyRelaxed(SlotMeta::HitFlag::SetTransform());
         }
         // All done.
         return h;
@@ -3317,8 +3289,7 @@ AutoHyperClockTable::HandleImpl* AutoHyperClockTable::Lookup(
 }
 
 void AutoHyperClockTable::Remove(HandleImpl* h) {
-  assert((h->meta.Load() >> ClockHandle::kStateShift) ==
-         ClockHandle::kStateConstruction);
+  assert(h->meta.Load().IsUnderConstruction());
 
   const HandleImpl& c_h = *h;
   PurgeImpl(&c_h.hashed_key);
@@ -3326,26 +3297,23 @@ void AutoHyperClockTable::Remove(HandleImpl* h) {
 
 bool AutoHyperClockTable::TryEraseHandle(HandleImpl* h, bool holding_ref,
                                          bool mark_invisible) {
-  uint64_t meta;
-  if (mark_invisible) {
-    // Set invisible
-    meta = h->meta.FetchAnd(
-        ~(uint64_t{ClockHandle::kStateVisibleBit} << ClockHandle::kStateShift));
-    // To local variable also
-    meta &=
-        ~(uint64_t{ClockHandle::kStateVisibleBit} << ClockHandle::kStateShift);
-  } else {
-    meta = h->meta.Load();
-  }
+  SlotMeta meta = h->meta.Load();
+  assert(!holding_ref || meta.IsShareable());
 
-  // Take ownership if no other refs
+  // Take ownership if no other refs, or set invisible if other refs exist (and
+  // mark_invisible is set).
+  SlotMeta construction_meta;
+  construction_meta.SetUnderConstruction();
   do {
-    if (GetRefcount(meta) != uint64_t{holding_ref}) {
+    if (meta.GetRefcount() != uint32_t{holding_ref}) {
       // Not last ref at some point in time during this call
+      if (mark_invisible) {
+        // Set invisible
+        h->meta.Apply(SlotMeta::VisibleFlag::ClearTransform());
+      }
       return false;
     }
-    if ((meta & (uint64_t{ClockHandle::kStateShareableBit}
-                 << ClockHandle::kStateShift)) == 0) {
+    if (!meta.IsShareable()) {
       // Someone else took ownership
       return false;
     }
@@ -3353,8 +3321,7 @@ bool AutoHyperClockTable::TryEraseHandle(HandleImpl* h, bool holding_ref,
     // another thread replaces this entry with another, reaches zero refs, and
     // then we end up erasing that other entry. That's an acceptable risk /
     // imprecision.
-  } while (!h->meta.CasWeak(meta, uint64_t{ClockHandle::kStateConstruction}
-                                      << ClockHandle::kStateShift));
+  } while (!h->meta.CasWeak(meta, construction_meta));
   // Took ownership
   // TODO? Delay freeing?
   h->FreeData(allocator_);
@@ -3381,27 +3348,24 @@ bool AutoHyperClockTable::Release(HandleImpl* h, bool useful,
   // is needed) and Erase. We do this to avoid an extra atomic read of the
   // variable usage_.
 
-  uint64_t old_meta;
+  SlotMeta old_meta;
   if (useful) {
     // Increment release counter to indicate was used
-    old_meta = h->meta.FetchAdd(ClockHandle::kReleaseIncrement);
+    auto add_release = ReleaseCounter::PlusTransformPromiseNoOverflow(1);
+    h->meta.Apply(add_release, &old_meta);
     // Correct for possible (but rare) overflow
     CorrectNearOverflow(old_meta, h->meta);
   } else {
     // Decrement acquire counter to pretend it never happened
-    old_meta = h->meta.FetchSub(ClockHandle::kAcquireIncrement);
+    auto sub_acquire = AcquireCounter::MinusTransformPromiseNoUnderflow(1);
+    h->meta.Apply(sub_acquire, &old_meta);
   }
 
-  assert((old_meta >> ClockHandle::kStateShift) &
-         ClockHandle::kStateShareableBit);
+  assert(old_meta.IsShareable());
   // No underflow
-  assert(((old_meta >> ClockHandle::kAcquireCounterShift) &
-          ClockHandle::kCounterMask) !=
-         ((old_meta >> ClockHandle::kReleaseCounterShift) &
-          ClockHandle::kCounterMask));
+  assert(old_meta.GetAcquireCounter() != old_meta.GetReleaseCounter());
 
-  if ((erase_if_last_ref || UNLIKELY(old_meta >> ClockHandle::kStateShift ==
-                                     ClockHandle::kStateInvisible))) {
+  if ((erase_if_last_ref || UNLIKELY(old_meta.IsInvisible()))) {
     // FIXME: There's a chance here that another thread could replace this
     // entry and we end up erasing the wrong one.
     return TryEraseHandle(h, /*holding_ref=*/false, /*mark_invisible=*/false);
@@ -3411,7 +3375,7 @@ bool AutoHyperClockTable::Release(HandleImpl* h, bool useful,
 }
 
 #ifndef NDEBUG
-void AutoHyperClockTable::TEST_ReleaseN(HandleImpl* h, size_t n) {
+void AutoHyperClockTable::TEST_ReleaseN(HandleImpl* h, uint32_t n) {
   if (n > 0) {
     // Do n-1 simple releases first
     TEST_ReleaseNMinus1(h, n);
@@ -3441,20 +3405,20 @@ void AutoHyperClockTable::EraseUnRefEntries() {
   for (size_t i = 0; i < usable_size; i++) {
     HandleImpl& h = array_[i];
 
-    uint64_t old_meta = h.meta.LoadRelaxed();
-    if (old_meta & (uint64_t{ClockHandle::kStateShareableBit}
-                    << ClockHandle::kStateShift) &&
-        GetRefcount(old_meta) == 0 &&
-        h.meta.CasStrong(old_meta, uint64_t{ClockHandle::kStateConstruction}
-                                       << ClockHandle::kStateShift)) {
-      // Took ownership
-      h.FreeData(allocator_);
-      usage_.FetchSubRelaxed(h.total_charge);
-      // NOTE: could be more efficient with a dedicated variant of
-      // PurgeImpl, but this is not a common operation
-      Remove(&h);
-      MarkEmpty(h);
-      occupancy_.FetchSub(1U);
+    SlotMeta old_meta = h.meta.LoadRelaxed();
+    if (old_meta.IsShareable() && old_meta.GetRefcount() == 0) {
+      SlotMeta construction_meta;
+      construction_meta.SetUnderConstruction();
+      if (h.meta.CasStrong(old_meta, construction_meta)) {
+        // Took ownership
+        h.FreeData(allocator_);
+        usage_.FetchSubRelaxed(h.total_charge);
+        // NOTE: could be more efficient with a dedicated variant of
+        // PurgeImpl, but this is not a common operation
+        Remove(&h);
+        MarkEmpty(h);
+        occupancy_.FetchSub(1U);
+      }
     }
   }
 }