mesh-rb 0.0.1 → 0.0.2

data/ext/mesh/mesh/src/meshable_arena.h

@@ -1,309 +0,0 @@
-// -*- mode: c++; c-basic-offset: 2; indent-tabs-mode: nil -*-
-// Copyright 2019 The Mesh Authors. All rights reserved.
-// Use of this source code is governed by the Apache License,
-// Version 2.0, that can be found in the LICENSE file.
-
-#pragma once
-#ifndef MESH_MESHABLE_ARENA_H
-#define MESH_MESHABLE_ARENA_H
-
-#if defined(_WIN32)
-#error "TODO"
-#include <windows.h>
-#else
-// UNIX
-#include <fcntl.h>
-#include <stdlib.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-#endif
-
-#if defined(__APPLE__) || defined(__FreeBSD__)
-#include <copyfile.h>
-#else
-#include <sys/sendfile.h>
-#endif
-
-#include <new>
-
-#include "internal.h"
-
-#include "cheap_heap.h"
-
-#include "bitmap.h"
-
-#include "mmap_heap.h"
-
-#ifndef MADV_DONTDUMP
-#define MADV_DONTDUMP 0
-#endif
-
-#ifndef MADV_DODUMP
-#define MADV_DODUMP 0
-#endif
-
-namespace mesh {
-
-class MeshableArena : public mesh::OneWayMmapHeap {
-private:
-  DISALLOW_COPY_AND_ASSIGN(MeshableArena);
-  typedef OneWayMmapHeap SuperHeap;
-
-public:
-  enum { Alignment = kPageSize };
-
-  explicit MeshableArena();
-
-  inline bool contains(const void *ptr) const {
-    auto arena = reinterpret_cast<uintptr_t>(_arenaBegin);
-    auto ptrval = reinterpret_cast<uintptr_t>(ptr);
-    return arena <= ptrval && ptrval < arena + kArenaSize;
-  }
-
-  char *pageAlloc(Span &result, size_t pageCount, size_t pageAlignment = 1);
-
-  void free(void *ptr, size_t sz, internal::PageType type);
-
-  inline void trackMiniHeap(const Span span, MiniHeapID id) {
-    // now that we know they are available, set the empty pages to
-    // in-use.  This is safe because this whole function is called
-    // under the GlobalHeap lock, so there is no chance of concurrent
-    // modification between the loop above and the one below.
-    for (size_t i = 0; i < span.length; i++) {
-#ifndef NDEBUG
-      d_assert(!_mhIndex[span.offset + i].load(std::memory_order_acquire).hasValue());
-      // auto mh = reinterpret_cast<MiniHeap *>(miniheapForArenaOffset(span.offset + i));
-      // mh->dumpDebug();
-#endif
-      setIndex(span.offset + i, id);
-    }
-  }
-
-  inline void *ATTRIBUTE_ALWAYS_INLINE miniheapForArenaOffset(Offset arenaOff) const {
-    const MiniHeapID mhOff = _mhIndex[arenaOff].load(std::memory_order_acquire);
-    if (likely(mhOff.hasValue())) {
-      return _mhAllocator.ptrFromOffset(mhOff.value());
-    }
-
-    return nullptr;
-  }
-
-  inline void *ATTRIBUTE_ALWAYS_INLINE lookupMiniheap(const void *ptr) const {
-    if (unlikely(!contains(ptr))) {
-      return nullptr;
-    }
-
-    // we've already checked contains, so we know this offset is
-    // within bounds
-    const auto arenaOff = offsetFor(ptr);
-    return miniheapForArenaOffset(arenaOff);
-  }
-
-  void beginMesh(void *keep, void *remove, size_t sz);
-  void finalizeMesh(void *keep, void *remove, size_t sz);
-
-  inline bool aboveMeshThreshold() const {
-    return _meshedPageCount > _maxMeshCount;
-  }
-
-  inline void setMaxMeshCount(size_t maxMeshCount) {
-    // debug("setting max map count: %zu", maxMeshCount);
-    _maxMeshCount = maxMeshCount;
-  }
-
-  inline size_t maxMeshCount() const {
-    return _maxMeshCount;
-  }
-
-  // protected:
-  // public for testing
-  void scavenge(bool force);
-  // like a scavenge, but we only MADV_FREE
-  void partialScavenge();
-
-  // return the maximum number of pages we've had meshed (and thus our
-  // savings) at any point in time.
-  inline size_t meshedPageHighWaterMark() const {
-    return _meshedPageCountHWM;
-  }
-
-  inline size_t RSSAtHighWaterMark() const {
-    return _rssKbAtHWM;
-  }
-
-  char *arenaBegin() const {
-    return reinterpret_cast<char *>(_arenaBegin);
-  }
-  void *arenaEnd() const {
-    return reinterpret_cast<char *>(_arenaBegin) + kArenaSize;
-  }
-
-  void doAfterForkChild();
-
-  void freePhys(void *ptr, size_t sz);
-
-private:
-  void expandArena(size_t minPagesAdded);
-  bool findPages(size_t pageCount, Span &result, internal::PageType &type);
-  bool ATTRIBUTE_NEVER_INLINE findPagesInner(internal::vector<Span> freeSpans[kSpanClassCount], size_t i,
-                                             size_t pageCount, Span &result);
-  Span reservePages(size_t pageCount, size_t pageAlignment);
-  internal::RelaxedBitmap allocatedBitmap(bool includeDirty = true) const;
-
-  void *malloc(size_t sz) = delete;
-
-  inline bool isAligned(const Span &span, const size_t pageAlignment) const {
-    return ptrvalFromOffset(span.offset) % (pageAlignment * kPageSize) == 0;
-  }
-
-  static constexpr size_t indexSize() {
-    // one pointer per page in our arena
-    return sizeof(Offset) * (kArenaSize / kPageSize);
-  }
-
-  inline void clearIndex(const Span &span) {
-    for (size_t i = 0; i < span.length; i++) {
-      // clear the miniheap pointers we were tracking
-      setIndex(span.offset + i, MiniHeapID{0});
-    }
-  }
-
-  inline void freeSpan(const Span &span, const internal::PageType flags) {
-    if (span.length == 0) {
-      return;
-    }
-
-    // this happens when we are trying to get an aligned allocation
-    // and returning excess back to the arena
-    if (flags == internal::PageType::Clean) {
-      _clean[span.spanClass()].push_back(span);
-      return;
-    }
-
-    clearIndex(span);
-
-    if (flags == internal::PageType::Dirty) {
-      if (kAdviseDump) {
-        madvise(ptrFromOffset(span.offset), span.length * kPageSize, MADV_DONTDUMP);
-      }
-      d_assert(span.length > 0);
-      _dirty[span.spanClass()].push_back(span);
-      _dirtyPageCount += span.length;
-
-      if (_dirtyPageCount > kMaxDirtyPageThreshold) {
-        // do a full scavenge with a probability 1/10
-        if (_fastPrng.inRange(0, 9) == 9) {
-          scavenge(true);
-        } else {
-          partialScavenge();
-        }
-      }
-    } else if (flags == internal::PageType::Meshed) {
-      // delay restoring the identity mapping
-      _toReset.push_back(span);
-    }
-  }
-
-  int openShmSpanFile(size_t sz);
-  int openSpanFile(size_t sz);
-  char *openSpanDir(int pid);
-
-  // pointer must already have been checked by `contains()` for bounds
-  inline Offset offsetFor(const void *ptr) const {
-    const uintptr_t ptrval = reinterpret_cast<uintptr_t>(ptr);
-    const uintptr_t arena = reinterpret_cast<uintptr_t>(_arenaBegin);
-
-    d_assert(ptrval >= arena);
-
-    return (ptrval - arena) / kPageSize;
-  }
-
-  inline uintptr_t ptrvalFromOffset(size_t off) const {
-    return reinterpret_cast<uintptr_t>(_arenaBegin) + off * kPageSize;
-  }
-
-  inline void *ptrFromOffset(size_t off) const {
-    return reinterpret_cast<void *>(ptrvalFromOffset(off));
-  }
-
-  inline void setIndex(size_t off, MiniHeapID val) {
-    d_assert(off < indexSize());
-    _mhIndex[off].store(val, std::memory_order_release);
-  }
-
-  static void staticAtExit();
-  static void staticPrepareForFork();
-  static void staticAfterForkParent();
-  static void staticAfterForkChild();
-
-  void exit() {
-    // FIXME: do this from the destructor, and test that destructor is
-    // called.  Also don't leak _spanDir
-    if (_spanDir != nullptr) {
-      rmdir(_spanDir);
-      _spanDir = nullptr;
-    }
-  }
-
-  inline void trackMeshed(const Span &span) {
-    for (size_t i = 0; i < span.length; i++) {
-      // this may already be 1 if it was a meshed virtual span that is
-      // now being re-meshed to a new owning miniheap
-      _meshedBitmap.tryToSet(span.offset + i);
-    }
-  }
-
-  inline void untrackMeshed(const Span &span) {
-    for (size_t i = 0; i < span.length; i++) {
-      d_assert(_meshedBitmap.isSet(span.offset + i));
-      _meshedBitmap.unset(span.offset + i);
-    }
-  }
-
-  inline void resetSpanMapping(const Span &span) {
-    auto ptr = ptrFromOffset(span.offset);
-    auto sz = span.byteLength();
-    mmap(ptr, sz, HL_MMAP_PROTECTION_MASK, kMapShared | MAP_FIXED, _fd, span.offset * kPageSize);
-  }
-
-  void prepareForFork();
-  void afterForkParent();
-  void afterForkChild();
-
-  void *_arenaBegin{nullptr};
-  // indexed by page offset.
-  atomic<MiniHeapID> *_mhIndex{nullptr};
-
-protected:
-  CheapHeap<64, kArenaSize / kPageSize> _mhAllocator{};
-  MWC _fastPrng;
-
-private:
-  Offset _end{};  // in pages
-
-  // spans that had been meshed, have been freed, and need to be reset
-  // to identity mappings in the page tables.
-  internal::vector<Span> _toReset;
-
-  internal::vector<Span> _clean[kSpanClassCount];
-  internal::vector<Span> _dirty[kSpanClassCount];
-
-  size_t _dirtyPageCount{0};
-
-  internal::RelaxedBitmap _meshedBitmap{
-      kArenaSize / kPageSize,
-      reinterpret_cast<char *>(OneWayMmapHeap().malloc(bitmap::representationSize(kArenaSize / kPageSize))), false};
-  size_t _meshedPageCount{0};
-  size_t _meshedPageCountHWM{0};
-  size_t _rssKbAtHWM{0};
-  size_t _maxMeshCount{kDefaultMaxMeshCount};
-
-  int _fd;
-  int _forkPipe[2]{-1, -1};  // used for signaling during fork
-  char *_spanDir{nullptr};
-};
-}  // namespace mesh
-
-#endif  // MESH_MESHABLE_ARENA_H

data/ext/mesh/mesh/src/meshing.h

@@ -1,60 +0,0 @@
-// -*- mode: c++; c-basic-offset: 2; indent-tabs-mode: nil -*-
-// Copyright 2019 The Mesh Authors. All rights reserved.
-// Use of this source code is governed by the Apache License,
-// Version 2.0, that can be found in the LICENSE file.
-
-#pragma once
-#ifndef MESH_MESHING_H
-#define MESH_MESHING_H
-
-#include <algorithm>
-#include <atomic>
-#include <limits>
-
-#include "bitmap.h"
-#include "common.h"
-#include "internal.h"
-#include "mini_heap.h"
-
-namespace mesh {
-
-using internal::Bitmap;
-
-inline bool bitmapsMeshable(const Bitmap::word_t *__restrict__ bitmap1, const Bitmap::word_t *__restrict__ bitmap2,
-                            size_t byteLen) noexcept {
-  d_assert(reinterpret_cast<uintptr_t>(bitmap1) % 16 == 0);
-  d_assert(reinterpret_cast<uintptr_t>(bitmap2) % 16 == 0);
-  d_assert(byteLen >= 8);
-  d_assert(byteLen % 8 == 0);
-
-  // because we hold the global lock (so no miniheap can transition
-  // from 'free' to 'attached'), the only possible data race we have
-  // here is our read with a write changing a bit from 1 -> 0.  That
-  // makes this race benign - we may have a false positive if an
-  // allocation is freed (it may cause 2 bitmaps to look like they
-  // overlap when _now_ they don't actually), but it won't cause
-  // correctness issues.
-  const auto bitmapL = (const uint64_t *)__builtin_assume_aligned(bitmap1, 16);
-  const auto bitmapR = (const uint64_t *)__builtin_assume_aligned(bitmap2, 16);
-
-  uint64_t result = 0;
-
-  for (size_t i = 0; i < byteLen / sizeof(size_t); i++) {
-    result |= bitmapL[i] & bitmapR[i];
-  }
-
-  return result == 0;
-}
-
-namespace method {
-
-// split miniheaps into two lists in a random order
-void halfSplit(MWC &prng, MiniHeapListEntry *miniheaps, SplitArray &left, size_t &leftSize, SplitArray &right,
-               size_t &rightSize) noexcept;
-
-void shiftedSplitting(MWC &prng, MiniHeapListEntry *miniheaps, SplitArray &left, SplitArray &right,
-                      const function<bool(std::pair<MiniHeap *, MiniHeap *> &&)> &meshFound) noexcept;
-}  // namespace method
-}  // namespace mesh
-
-#endif  // MESH_MESHING_H

data/ext/mesh/mesh/src/mini_heap.h

@@ -1,532 +0,0 @@
-// -*- mode: c++; c-basic-offset: 2; indent-tabs-mode: nil -*-
-// Copyright 2019 The Mesh Authors. All rights reserved.
-// Use of this source code is governed by the Apache License,
-// Version 2.0, that can be found in the LICENSE file.
-
-#pragma once
-#ifndef MESH_MINI_HEAP_H
-#define MESH_MINI_HEAP_H
-
-#include <pthread.h>
-
-#include <atomic>
-#include <random>
-
-#include "bitmap.h"
-#include "fixed_array.h"
-#include "internal.h"
-
-#include "rng/mwc.h"
-
-#include "heaplayers.h"
-
-namespace mesh {
-
-class MiniHeap;
-
-class Flags {
-private:
-  DISALLOW_COPY_AND_ASSIGN(Flags);
-
-  static inline constexpr uint32_t ATTRIBUTE_ALWAYS_INLINE getSingleBitMask(uint32_t pos) {
-    return 1UL << pos;
-  }
-  static constexpr uint32_t SizeClassShift = 0;
-  static constexpr uint32_t FreelistIdShift = 6;
-  static constexpr uint32_t ShuffleVectorOffsetShift = 8;
-  static constexpr uint32_t MaxCountShift = 16;
-  static constexpr uint32_t MeshedOffset = 30;
-
-  inline void ATTRIBUTE_ALWAYS_INLINE setMasked(uint32_t mask, uint32_t newVal) {
-    uint32_t oldFlags = _flags.load(std::memory_order_relaxed);
-    while (!atomic_compare_exchange_weak_explicit(&_flags,
-                                                  &oldFlags,                   // old val
-                                                  (oldFlags & mask) | newVal,  // new val
-                                                  std::memory_order_release,   // success mem model
-                                                  std::memory_order_relaxed)) {
-    }
-  }
-
-public:
-  explicit Flags(uint32_t maxCount, uint32_t sizeClass, uint32_t svOffset, uint32_t freelistId) noexcept
-      : _flags{(maxCount << MaxCountShift) + (sizeClass << SizeClassShift) + (svOffset << ShuffleVectorOffsetShift) +
-               (freelistId << FreelistIdShift)} {
-    d_assert((freelistId & 0x3) == freelistId);
-    d_assert((sizeClass & ((1 << FreelistIdShift) - 1)) == sizeClass);
-    d_assert(svOffset < 255);
-    d_assert_msg(sizeClass < 255, "sizeClass: %u", sizeClass);
-    d_assert(maxCount <= 256);
-    d_assert(this->maxCount() == maxCount);
-  }
-
-  inline uint32_t freelistId() const {
-    return (_flags.load(std::memory_order_seq_cst) >> FreelistIdShift) & 0x3;
-  }
-
-  inline void setFreelistId(uint32_t freelistId) {
-    static_assert(list::Max <= (1 << FreelistIdShift), "expected max < 4");
-    d_assert(freelistId < list::Max);
-    uint32_t mask = ~(static_cast<uint32_t>(0x3) << FreelistIdShift);
-    uint32_t newVal = (static_cast<uint32_t>(freelistId) << FreelistIdShift);
-    setMasked(mask, newVal);
-  }
-
-  inline uint32_t maxCount() const {
-    // XXX: does this assume little endian?
-    return (_flags.load(std::memory_order_seq_cst) >> MaxCountShift) & 0x1ff;
-  }
-
-  inline uint32_t sizeClass() const {
-    return (_flags.load(std::memory_order_seq_cst) >> SizeClassShift) & 0x3f;
-  }
-
-  inline uint8_t svOffset() const {
-    return (_flags.load(std::memory_order_seq_cst) >> ShuffleVectorOffsetShift) & 0xff;
-  }
-
-  inline void setSvOffset(uint8_t off) {
-    d_assert(off < 255);
-    uint32_t mask = ~(static_cast<uint32_t>(0xff) << ShuffleVectorOffsetShift);
-    uint32_t newVal = (static_cast<uint32_t>(off) << ShuffleVectorOffsetShift);
-    setMasked(mask, newVal);
-  }
-
-  inline void setMeshed() {
-    set(MeshedOffset);
-  }
-
-  inline void unsetMeshed() {
-    unset(MeshedOffset);
-  }
-
-  inline bool ATTRIBUTE_ALWAYS_INLINE isMeshed() const {
-    return is(MeshedOffset);
-  }
-
-private:
-  inline bool ATTRIBUTE_ALWAYS_INLINE is(size_t offset) const {
-    const auto mask = getSingleBitMask(offset);
-    return (_flags.load(std::memory_order_acquire) & mask) == mask;
-  }
-
-  inline void set(size_t offset) {
-    const uint32_t mask = getSingleBitMask(offset);
-
-    uint32_t oldFlags = _flags.load(std::memory_order_relaxed);
-    while (!atomic_compare_exchange_weak_explicit(&_flags,
-                                                  &oldFlags,                  // old val
-                                                  oldFlags | mask,            // new val
-                                                  std::memory_order_release,  // success mem model
-                                                  std::memory_order_relaxed)) {
-    }
-  }
-
-  inline void unset(size_t offset) {
-    const uint32_t mask = getSingleBitMask(offset);
-
-    uint32_t oldFlags = _flags.load(std::memory_order_relaxed);
-    while (!atomic_compare_exchange_weak_explicit(&_flags,
-                                                  &oldFlags,                  // old val
-                                                  oldFlags & ~mask,           // new val
-                                                  std::memory_order_release,  // success mem model
-                                                  std::memory_order_relaxed)) {
-    }
-  }
-
-  std::atomic<uint32_t> _flags;
-};
-
-class MiniHeap {
-private:
-  DISALLOW_COPY_AND_ASSIGN(MiniHeap);
-
-public:
-  MiniHeap(void *arenaBegin, Span span, size_t objectCount, size_t objectSize)
-      : _bitmap(objectCount),
-        _span(span),
-        _flags(objectCount, objectCount > 1 ? SizeMap::SizeClass(objectSize) : 1, 0, list::Attached),
-        _objectSizeReciprocal(1.0 / (float)objectSize) {
-    // debug("sizeof(MiniHeap): %zu", sizeof(MiniHeap));
-
-    d_assert(_bitmap.inUseCount() == 0);
-
-    const auto expectedSpanSize = _span.byteLength();
-    d_assert_msg(expectedSpanSize == spanSize(), "span size %zu == %zu (%u, %u)", expectedSpanSize, spanSize(),
-                 maxCount(), this->objectSize());
-
-    // d_assert_msg(spanSize == static_cast<size_t>(_spanSize), "%zu != %hu", spanSize, _spanSize);
-    // d_assert_msg(objectSize == static_cast<size_t>(objectSize()), "%zu != %hu", objectSize, _objectSize);
-
-    d_assert(!_nextMeshed.hasValue());
-
-    // debug("new:\n");
-    // dumpDebug();
-  }
-
-  ~MiniHeap() {
-    // debug("destruct:\n");
-    // dumpDebug();
-  }
-
-  inline Span span() const {
-    return _span;
-  }
-
-  void printOccupancy() const {
-    mesh::debug("{\"name\": \"%p\", \"object-size\": %d, \"length\": %d, \"mesh-count\": %d, \"bitmap\": \"%s\"}\n",
-                this, objectSize(), maxCount(), meshCount(), _bitmap.to_string(maxCount()).c_str());
-  }
-
-  inline void ATTRIBUTE_ALWAYS_INLINE free(void *arenaBegin, void *ptr) {
-    // the logic in globalFree is
-    // updated to allow the 'race' between lock-free freeing and
-    // meshing
-    // d_assert(!isMeshed());
-    const ssize_t off = getOff(arenaBegin, ptr);
-    if (unlikely(off < 0)) {
-      d_assert(false);
-      return;
-    }
-
-    freeOff(off);
-  }
-
-  inline bool clearIfNotFree(void *arenaBegin, void *ptr) {
-    const ssize_t off = getOff(arenaBegin, ptr);
-    const auto notWasSet = _bitmap.unset(off);
-    const auto wasSet = !notWasSet;
-    return wasSet;
-  }
-
-  inline void ATTRIBUTE_ALWAYS_INLINE freeOff(size_t off) {
-    d_assert_msg(_bitmap.isSet(off), "MiniHeap(%p) expected bit %zu to be set (svOff:%zu)", this, off, svOffset());
-    _bitmap.unset(off);
-  }
-
-  /// Copies (for meshing) the contents of src into our span.
-  inline void consume(const void *arenaBegin, MiniHeap *src) {
-    // this would be bad
-    d_assert(src != this);
-    d_assert(objectSize() == src->objectSize());
-
-    src->setMeshed();
-    const auto srcSpan = src->getSpanStart(arenaBegin);
-    const auto objectSize = this->objectSize();
-
-    // this both avoids the need to call `freeOff` in the loop
-    // below, but it ensures we will be able to check for bitmap
-    // setting races in GlobalHeap::freeFor
-    const auto srcBitmap = src->takeBitmap();
-
-    // for each object in src, copy it to our backing span + update
-    // our bitmap and in-use count
-    for (auto const &off : srcBitmap) {
-      d_assert(off < maxCount());
-      d_assert(!_bitmap.isSet(off));
-
-      void *srcObject = reinterpret_cast<void *>(srcSpan + off * objectSize);
-      // need to ensure we update the bitmap and in-use count
-      void *dstObject = mallocAt(arenaBegin, off);
-      // debug("meshing: %zu (%p <- %p)\n", off, dstObject, srcObject);
-      d_assert(dstObject != nullptr);
-      memcpy(dstObject, srcObject, objectSize);
-      // debug("\t'%s'\n", dstObject);
-      // debug("\t'%s'\n", srcObject);
-    }
-
-    trackMeshedSpan(GetMiniHeapID(src));
-  }
-
-  inline size_t spanSize() const {
-    return _span.byteLength();
-  }
-
-  inline uint32_t ATTRIBUTE_ALWAYS_INLINE maxCount() const {
-    return _flags.maxCount();
-  }
-
-  inline bool ATTRIBUTE_ALWAYS_INLINE isLargeAlloc() const {
-    return maxCount() == 1;
-  }
-
-  inline size_t objectSize() const {
-    if (likely(!isLargeAlloc())) {
-      // this doesn't handle all the corner cases of roundf(3),
-      // but it does work for all of our small object size classes
-      return static_cast<size_t>(1 / _objectSizeReciprocal + 0.5);
-    } else {
-      return _span.length * kPageSize;
-    }
-  }
-
-  inline int sizeClass() const {
-    return _flags.sizeClass();
-  }
-
-  inline uintptr_t getSpanStart(const void *arenaBegin) const {
-    const auto beginval = reinterpret_cast<uintptr_t>(arenaBegin);
-    return beginval + _span.offset * kPageSize;
-  }
-
-  inline bool ATTRIBUTE_ALWAYS_INLINE isEmpty() const {
-    return _bitmap.inUseCount() == 0;
-  }
-
-  inline bool ATTRIBUTE_ALWAYS_INLINE isFull() const {
-    return _bitmap.inUseCount() == maxCount();
-  }
-
-  inline uint32_t ATTRIBUTE_ALWAYS_INLINE inUseCount() const {
-    return _bitmap.inUseCount();
-  }
-
-  inline size_t bytesFree() const {
-    return inUseCount() * objectSize();
-  }
-
-  inline void setMeshed() {
-    _flags.setMeshed();
-  }
-
-  inline void setAttached(pid_t current, MiniHeapListEntry *listHead) {
-    // mesh::debug("MiniHeap(%p:%5zu): current <- %u\n", this, objectSize(), current);
-    _current.store(current, std::memory_order::memory_order_release);
-    if (listHead != nullptr) {
-      _freelist.remove(listHead);
-    }
-    this->setFreelistId(list::Attached);
-  }
-
-  inline uint8_t svOffset() const {
-    return _flags.svOffset();
-  }
-
-  inline void setSvOffset(uint8_t off) {
-    // debug("MiniHeap(%p) SET svOff:%zu)", this, off);
-    _flags.setSvOffset(off);
-  }
-
-  inline uint8_t freelistId() const {
-    return _flags.freelistId();
-  }
-
-  inline void setFreelistId(uint8_t id) {
-    _flags.setFreelistId(id);
-  }
-
-  inline pid_t current() const {
-    return _current.load(std::memory_order::memory_order_acquire);
-  }
-
-  inline void unsetAttached() {
-    // mesh::debug("MiniHeap(%p:%5zu): current <- UNSET\n", this, objectSize());
-    _current.store(0, std::memory_order::memory_order_release);
-  }
-
-  inline bool isAttached() const {
-    return current() != 0;
-  }
-
-  inline bool ATTRIBUTE_ALWAYS_INLINE isMeshed() const {
-    return _flags.isMeshed();
-  }
-
-  inline bool ATTRIBUTE_ALWAYS_INLINE hasMeshed() const {
-    return _nextMeshed.hasValue();
-  }
-
-  inline bool isMeshingCandidate() const {
-    return !isAttached() && objectSize() < kPageSize;
-  }
-
-  /// Returns the fraction full (in the range [0, 1]) that this miniheap is.
-  inline double fullness() const {
-    return static_cast<double>(inUseCount()) / static_cast<double>(maxCount());
-  }
-
-  internal::RelaxedFixedBitmap takeBitmap() {
-    const auto capacity = this->maxCount();
-    internal::RelaxedFixedBitmap zero{capacity};
-    internal::RelaxedFixedBitmap result{capacity};
-    _bitmap.setAndExchangeAll(result.mut_bits(), zero.bits());
-    return result;
-  }
-
-  const internal::Bitmap &bitmap() const {
-    return _bitmap;
-  }
-
-  internal::Bitmap &writableBitmap() {
-    return _bitmap;
-  }
-
-  void trackMeshedSpan(MiniHeapID id) {
-    hard_assert(id.hasValue());
-
-    if (!_nextMeshed.hasValue()) {
-      _nextMeshed = id;
-    } else {
-      GetMiniHeap(_nextMeshed)->trackMeshedSpan(id);
-    }
-  }
-
-public:
-  template <class Callback>
-  inline void forEachMeshed(Callback cb) const {
-    if (cb(this))
-      return;
-
-    if (_nextMeshed.hasValue()) {
-      const auto mh = GetMiniHeap(_nextMeshed);
-      mh->forEachMeshed(cb);
-    }
-  }
-
-  template <class Callback>
-  inline void forEachMeshed(Callback cb) {
-    if (cb(this))
-      return;
-
-    if (_nextMeshed.hasValue()) {
-      auto mh = GetMiniHeap(_nextMeshed);
-      mh->forEachMeshed(cb);
-    }
-  }
-
-  bool isRelated(MiniHeap *other) const {
-    auto otherFound = false;
-    this->forEachMeshed([&](const MiniHeap *eachMh) {
-      const auto found = eachMh == other;
-      otherFound = found;
-      return found;
-    });
-    return otherFound;
-  }
-
-  size_t meshCount() const {
-    size_t count = 0;
-
-    const MiniHeap *mh = this;
-    while (mh != nullptr) {
-      count++;
-
-      auto next = mh->_nextMeshed;
-      mh = next.hasValue() ? GetMiniHeap(next) : nullptr;
-    }
-
-    return count;
-  }
-
-  MiniHeapListEntry *getFreelist() {
-    return &_freelist;
-  }
-
-  /// public for meshTest only
-  inline void *mallocAt(const void *arenaBegin, size_t off) {
-    if (!_bitmap.tryToSet(off)) {
-      mesh::debug("%p: MA %u", this, off);
-      dumpDebug();
-      return nullptr;
-    }
-
-    return ptrFromOffset(arenaBegin, off);
-  }
-
-  inline void *ptrFromOffset(const void *arenaBegin, size_t off) {
-    return reinterpret_cast<void *>(getSpanStart(arenaBegin) + off * objectSize());
-  }
-
-  inline bool operator<(MiniHeap *&rhs) noexcept {
-    return this->inUseCount() < rhs->inUseCount();
-  }
-
-  void dumpDebug() const {
-    const auto heapPages = spanSize() / HL::CPUInfo::PageSize;
-    const size_t inUseCount = this->inUseCount();
-    const size_t meshCount = this->meshCount();
-    mesh::debug(
-        "MiniHeap(%p:%5zu): %3zu objects on %2zu pages (inUse: %zu, spans: %zu)\t%p-%p\tFreelist{prev:%u, next:%u}\n",
-        this, objectSize(), maxCount(), heapPages, inUseCount, meshCount, _span.offset * kPageSize,
-        _span.offset * kPageSize + spanSize(), _freelist.prev(), _freelist.next());
-    mesh::debug("\t%s\n", _bitmap.to_string(maxCount()).c_str());
-  }
-
-  // this only works for unmeshed miniheaps
-  inline uint8_t ATTRIBUTE_ALWAYS_INLINE getUnmeshedOff(const void *arenaBegin, void *ptr) const {
-    const auto ptrval = reinterpret_cast<uintptr_t>(ptr);
-
-    uintptr_t span = reinterpret_cast<uintptr_t>(arenaBegin) + _span.offset * kPageSize;
-    d_assert(span != 0);
-
-    const size_t off = (ptrval - span) * _objectSizeReciprocal;
-    d_assert(off < maxCount());
-
-    return off;
-  }
-
-  inline uint8_t ATTRIBUTE_ALWAYS_INLINE getOff(const void *arenaBegin, void *ptr) const {
-    const auto span = spanStart(reinterpret_cast<uintptr_t>(arenaBegin), ptr);
-    d_assert(span != 0);
-    const auto ptrval = reinterpret_cast<uintptr_t>(ptr);
-
-    const size_t off = (ptrval - span) * _objectSizeReciprocal;
-    d_assert(off < maxCount());
-
-    return off;
-  }
-
-protected:
-  inline uintptr_t ATTRIBUTE_ALWAYS_INLINE spanStart(uintptr_t arenaBegin, void *ptr) const {
-    const auto ptrval = reinterpret_cast<uintptr_t>(ptr);
-    const auto len = _span.byteLength();
-
-    // manually unroll loop once to capture the common case of
-    // un-meshed miniheaps
-    uintptr_t spanptr = arenaBegin + _span.offset * kPageSize;
-    if (likely(spanptr <= ptrval && ptrval < spanptr + len)) {
-      return spanptr;
-    }
-
-    return spanStartSlowpath(arenaBegin, ptrval);
-  }
-
-  uintptr_t ATTRIBUTE_NEVER_INLINE spanStartSlowpath(uintptr_t arenaBegin, uintptr_t ptrval) const {
-    const auto len = _span.byteLength();
-    uintptr_t spanptr = 0;
-
-    const MiniHeap *mh = this;
-    while (true) {
-      if (unlikely(!mh->_nextMeshed.hasValue())) {
-        abort();
-      }
-
-      mh = GetMiniHeap(mh->_nextMeshed);
-
-      const uintptr_t meshedSpanptr = arenaBegin + mh->span().offset * kPageSize;
-      if (meshedSpanptr <= ptrval && ptrval < meshedSpanptr + len) {
-        spanptr = meshedSpanptr;
-        break;
-      }
-    };
-
-    return spanptr;
-  }
-
-  internal::Bitmap _bitmap;           // 32 bytes 32
-  const Span _span;                   // 8        40
-  MiniHeapListEntry _freelist{};      // 8        48
-  atomic<pid_t> _current{0};          // 4        52
-  Flags _flags;                       // 4        56
-  const float _objectSizeReciprocal;  // 4        60
-  MiniHeapID _nextMeshed{};           // 4        64
-};
-
-typedef FixedArray<MiniHeap, 63> MiniHeapArray;
-
-static_assert(sizeof(pid_t) == 4, "pid_t not 32-bits!");
-static_assert(sizeof(mesh::internal::Bitmap) == 32, "Bitmap too big!");
-static_assert(sizeof(MiniHeap) == 64, "MiniHeap too big!");
-static_assert(sizeof(MiniHeapArray) == 64 * sizeof(void *), "MiniHeapArray too big!");
-}  // namespace mesh
-
-#endif  // MESH_MINI_HEAP_H