mesh-rb 0.0.1 → 0.0.2

data/ext/mesh/mesh/src/testing/userfaultfd-kernel-copy.cc

@@ -1,164 +0,0 @@
-#include <fcntl.h>
-#include <linux/userfaultfd.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <sys/syscall.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-#include <cstdarg>
-#include <cstring>
-#include <thread>
-
-constexpr size_t kPageSize = 4096;
-constexpr size_t kDataLen = 128;
-constexpr size_t kArenaSize = kPageSize * 2;
-
-void __attribute__((noreturn)) die(const char *fmt, ...) {
-  va_list args;
-
-  va_start(args, fmt);
-  vfprintf(stderr, fmt, args);
-  va_end(args);
-
-  exit(EXIT_FAILURE);
-}
-
-void kernelCopy(int tmpFd, char *addr, char const *knownGood) {
-  errno = 0;
-  ssize_t len = read(tmpFd, addr, kDataLen);
-  if (len != kDataLen) {
-    fprintf(stderr, "read: %s (len: %zd)\n", strerror(errno), len);
-    return;
-  }
-
-  if (memcmp(knownGood, addr, kDataLen) != 0) {
-    die("data read from file doesn't match knownGood\n");
-  }
-
-  printf("read from file went as expected; data looks good.\n");
-}
-
-void setupPageTrap(int faultFd, char *arenaBegin, size_t pageOff, size_t len) {
-  char *mappingBegin = arenaBegin + pageOff * kPageSize;
-
-  // step 0: register the range with userfaultfd
-  struct uffdio_register ufRegister = {};
-  ufRegister.range.start = reinterpret_cast<unsigned long>(mappingBegin);
-  ufRegister.range.len = len;
-  ufRegister.mode = UFFDIO_REGISTER_MODE_WP;
-  if (ioctl(faultFd, UFFDIO_REGISTER, &ufRegister) == -1) {
-    die("ioctl(UFFDIO_REGISTER): %s\n", strerror(errno));
-  }
-
-  // step 1: use userfaultfd (rather than mprotect) to remap the pages read-only
-  struct uffdio_writeprotect ufWriteProtect = {};
-  ufWriteProtect.range.start = reinterpret_cast<unsigned long>(mappingBegin);
-  ufWriteProtect.range.len = len;
-  ufWriteProtect.mode = UFFDIO_WRITEPROTECT_MODE_WP;
-  if (ioctl(faultFd, UFFDIO_WRITEPROTECT, &ufWriteProtect) == -1) {
-    die("ioctl(UFFDIO_WRITEPROTECT): %s\n", strerror(errno));
-  }
-}
-
-int main() {
-  // get some random bytes + write it to a temp file
-  char data[kDataLen] = {};
-  if (getentropy(data, kDataLen) == -1) {
-    die("getentropy: %s\n", strerror(errno));
-  }
-
-  char tmpFilename[] = "/tmp/userfaultfd-test-XXXXXX";
-  int tmpFd = mkstemp(tmpFilename);
-  if (tmpFd == -1) {
-    die("mkstemp: %s\n", strerror(errno));
-  }
-  if (unlink(tmpFilename) == -1) {
-    die("unlink: %s\n", strerror(errno));
-  }
-
-  size_t len = write(tmpFd, data, kDataLen);
-  if (len < kDataLen) {
-    die("write: partial write of %d\n", len);
-  }
-  if (lseek(tmpFd, 0, SEEK_SET) != 0) {
-    die("lseek failed\n");
-  }
-
-  // the write-protection patchset doesn't yet support non-anonymous (or hugepage) VMAs
-  char *arenaBegin = reinterpret_cast<char *>(
-      mmap(nullptr, kArenaSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0));
-  if (arenaBegin == MAP_FAILED) {
-    die("mmap: %s\n", strerror(errno));
-  }
-
-  // pre-fault things in; the write protection stuff below doesn't seem to work
-  // well if the pages aren't already faulted in.  in a real allocator/GC the pages
-  // we're dealing with will be faulted in so this seems fine.
-  for (size_t i = 0; i < kArenaSize; i += kPageSize) {
-    arenaBegin[i] = 0;
-  }
-
-  int faultFd = static_cast<int>(syscall(__NR_userfaultfd, O_CLOEXEC));
-  if (faultFd == -1) {
-    die("userfaultfd: %s\n", strerror(errno));
-  }
-
-  // the only feature we care about is write-protection faults -- this will fail
-  // on a mainline kernel as the patchset isn't upstream yet.
-  struct uffdio_api ufApi = {};
-  ufApi.api = UFFD_API;
-  ufApi.features = UFFD_FEATURE_PAGEFAULT_FLAG_WP;
-  if (ioctl(faultFd, UFFDIO_API, &ufApi) == -1) {
-    die("ioctl(UFFDIO_API): %s\n", strerror(errno));
-  }
-  if (ufApi.api != UFFD_API) {
-    die("ioctl(UFFDIO_API) API version mismatch %lld != %lld\n", ufApi.api, UFFD_API);
-  }
-
-  // prefault in the entire arena to match the fact that the heap will already
-  // be faulted in when meshing pages together
-  for (size_t i = 0; i < kArenaSize; i += kPageSize) {
-    arenaBegin[i] = 0;
-  }
-
-  // write-protect part of the arena
-  setupPageTrap(faultFd, arenaBegin, 0, kPageSize);
-
-  printf("part of the arena is read-only now and writes raise an event on our userfaultfd\n");
-
-  // even works with kernelspace; +57 is to verify that non-page-aligned access work
-  std::thread t1(kernelCopy, tmpFd, arenaBegin + 57, data);
-
-  // now suspend until we receive an event from the kernelCopy thread
-  struct uffd_msg msg = {};
-  int nRead = read(faultFd, &msg, sizeof(msg));
-  if (nRead == 0) {
-    die("EOF on userfaultfd!\n");
-  } else if (nRead == -1) {
-    die("read: %s\n", strerror(errno));
-  }
-
-  if (msg.event != UFFD_EVENT_PAGEFAULT) {
-    die("Unexpected event on userfaultfd\n");
-  }
-
-  printf("    UFFD_EVENT_PAGEFAULT event: ");
-  printf("flags = %llx; ", msg.arg.pagefault.flags);
-  printf("address = %llx\n", msg.arg.pagefault.address);
-
-  sleep(2);
-
-  printf("removing write protection and waking up other thread\n");
-
-  // turn off write-protection; implicitly wakes the other thread up
-  struct uffdio_writeprotect ufWriteProtect = {};
-  ufWriteProtect.range.start = msg.arg.pagefault.address;
-  ufWriteProtect.range.len = kPageSize;
-  ufWriteProtect.mode = 0;
-  if (ioctl(faultFd, UFFDIO_WRITEPROTECT, &ufWriteProtect) == -1) {
-    die("ioctl(UFFDIO_WRITEPROTECT): %s\n", strerror(errno));
-  }
-
-  t1.join();
-}

data/ext/mesh/mesh/src/thread_local_heap.cc

@@ -1,163 +0,0 @@
-// -*- mode: c++; c-basic-offset: 2; indent-tabs-mode: nil -*-
-// Copyright 2019 The Mesh Authors. All rights reserved.
-// Use of this source code is governed by the Apache License,
-// Version 2.0, that can be found in the LICENSE file.
-
-#include "thread_local_heap.h"
-
-namespace mesh {
-
-#ifdef MESH_HAVE_TLS
-__thread ThreadLocalHeap *ThreadLocalHeap::_threadLocalHeap ATTR_INITIAL_EXEC CACHELINE_ALIGNED;
-#endif
-ThreadLocalHeap *ThreadLocalHeap::_threadLocalHeaps{nullptr};
-bool ThreadLocalHeap::_tlhInitialized{false};
-pthread_key_t ThreadLocalHeap::_heapKey{0};
-
-void ThreadLocalHeap::DestroyThreadLocalHeap(void *ptr) {
-  if (ptr != nullptr) {
-#ifdef MESH_HAVE_TLS
-    _threadLocalHeap = nullptr;
-#endif
-    DeleteHeap(reinterpret_cast<ThreadLocalHeap *>(ptr));
-  }
-}
-
-// similar to what TCMalloc does; the constructor initializes our
-// pthread key and does this in such a way that we aren't recursively
-// calling into pthread code (and deadlocking).
-class MeshMallocGuard {
-public:
-  MeshMallocGuard() {
-    ThreadLocalHeap::InitTLH();
-  }
-};
-
-static MeshMallocGuard module_initialization_hook;
-
-void ThreadLocalHeap::InitTLH() {
-  hard_assert(!_tlhInitialized);
-  pthread_key_create(&_heapKey, DestroyThreadLocalHeap);
-  _tlhInitialized = true;
-}
-
-ThreadLocalHeap *ThreadLocalHeap::NewHeap(pthread_t current) {
-  // we just allocate out of our internal heap
-  void *buf = mesh::internal::Heap().malloc(sizeof(ThreadLocalHeap));
-  static_assert(sizeof(ThreadLocalHeap) < 4096 * 8, "tlh should have a reasonable size");
-  hard_assert(buf != nullptr);
-  hard_assert(reinterpret_cast<uintptr_t>(buf) % CACHELINE_SIZE == 0);
-
-  auto heap = new (buf) ThreadLocalHeap(&mesh::runtime().heap(), current);
-
-  heap->_prev = nullptr;
-  heap->_next = _threadLocalHeaps;
-  if (_threadLocalHeaps != nullptr) {
-    _threadLocalHeaps->_prev = heap;
-  }
-  _threadLocalHeaps = heap;
-
-  return heap;
-}
-
-ThreadLocalHeap *ThreadLocalHeap::CreateHeapIfNecessary() {
-#ifdef MESH_HAVE_TLS
-  const bool maybeReentrant = !_tlhInitialized;
-  // check to see if we really need to create the heap
-  if (_tlhInitialized && _threadLocalHeap != nullptr) {
-    return _threadLocalHeap;
-  }
-#else
-  const bool maybeReentrant = true;
-#endif
-
-  ThreadLocalHeap *heap = nullptr;
-
-  {
-    std::lock_guard<GlobalHeap> lock(mesh::runtime().heap());
-
-    const pthread_t current = pthread_self();
-
-    if (maybeReentrant) {
-      for (ThreadLocalHeap *h = _threadLocalHeaps; h != nullptr; h = h->_next) {
-        if (h->_pthreadCurrent == current) {
-          heap = h;
-          break;
-        }
-      }
-    }
-
-    if (heap == nullptr) {
-      heap = NewHeap(current);
-    }
-  }
-
-  if (!heap->_inSetSpecific && _tlhInitialized) {
-    heap->_inSetSpecific = true;
-#ifdef MESH_HAVE_TLS
-    _threadLocalHeap = heap;
-#endif
-    pthread_setspecific(_heapKey, heap);
-    heap->_inSetSpecific = false;
-  }
-
-  return heap;
-}
-
-void ThreadLocalHeap::DeleteHeap(ThreadLocalHeap *heap) {
-  if (heap == nullptr) {
-    return;
-  }
-
-  // manually invoke the destructor
-  heap->ThreadLocalHeap::~ThreadLocalHeap();
-
-  if (heap->_next != nullptr) {
-    heap->_next->_prev = heap->_prev;
-  }
-  if (heap->_prev != nullptr) {
-    heap->_prev->_next = heap->_next;
-  }
-  if (_threadLocalHeaps == heap) {
-    _threadLocalHeaps = heap->_next;
-  }
-
-  mesh::internal::Heap().free(reinterpret_cast<void *>(heap));
-}
-
-void ThreadLocalHeap::releaseAll() {
-  for (size_t i = 1; i < kNumBins; i++) {
-    _shuffleVector[i].refillMiniheaps();
-    _global->releaseMiniheaps(_shuffleVector[i].miniheaps());
-  }
-}
-
-// we get here if the shuffleVector is exhausted
-void *CACHELINE_ALIGNED_FN ThreadLocalHeap::smallAllocSlowpath(size_t sizeClass) {
-  ShuffleVector &shuffleVector = _shuffleVector[sizeClass];
-
-  // we grab multiple MiniHeaps at a time from the global heap.  often
-  // it is possible to refill the freelist from a not-yet-used
-  // MiniHeap we already have, without global cross-thread
-  // synchronization
-  if (likely(shuffleVector.localRefill())) {
-    return shuffleVector.malloc();
-  }
-
-  return smallAllocGlobalRefill(shuffleVector, sizeClass);
-}
-
-void *CACHELINE_ALIGNED_FN ThreadLocalHeap::smallAllocGlobalRefill(ShuffleVector &shuffleVector, size_t sizeClass) {
-  const size_t sizeMax = SizeMap::ByteSizeForClass(sizeClass);
-
-  _global->allocSmallMiniheaps(sizeClass, sizeMax, shuffleVector.miniheaps(), _current);
-  shuffleVector.reinit();
-
-  d_assert(!shuffleVector.isExhausted());
-
-  void *ptr = shuffleVector.malloc();
-  d_assert(ptr != nullptr);
-
-  return ptr;
-}
-}  // namespace mesh

data/ext/mesh/mesh/src/thread_local_heap.h

@@ -1,268 +0,0 @@
-// -*- mode: c++; c-basic-offset: 2; indent-tabs-mode: nil -*-
-// Copyright 2019 The Mesh Authors. All rights reserved.
-// Use of this source code is governed by the Apache License,
-// Version 2.0, that can be found in the LICENSE file.
-
-#ifndef MESH_THREAD_LOCAL_HEAP_H
-#define MESH_THREAD_LOCAL_HEAP_H
-
-#if !defined(_WIN32)
-#include <pthread.h>
-#include <stdalign.h>
-#endif
-
-#include <sys/types.h>
-
-#include <algorithm>
-#include <atomic>
-
-#include "internal.h"
-#include "mini_heap.h"
-#include "shuffle_vector.h"
-
-#include "rng/mwc.h"
-
-#include "heaplayers.h"
-
-#include "runtime.h"
-
-using namespace HL;
-
-namespace mesh {
-
-class LocalHeapStats {
-public:
-  atomic_size_t allocCount{0};
-  atomic_size_t freeCount{0};
-};
-
-class ThreadLocalHeap {
-private:
-  DISALLOW_COPY_AND_ASSIGN(ThreadLocalHeap);
-
-public:
-  enum { Alignment = 16 };
-
-  ThreadLocalHeap(GlobalHeap *global, pthread_t pthreadCurrent)
-      : _current(gettid()),
-        _global(global),
-        _pthreadCurrent(pthreadCurrent),
-        _prng(internal::seed(), internal::seed()),
-        _maxObjectSize(SizeMap::ByteSizeForClass(kNumBins - 1)) {
-    const auto arenaBegin = _global->arenaBegin();
-    // when asked, give 16-byte allocations for 0-byte requests
-    _shuffleVector[0].initialInit(arenaBegin, SizeMap::ByteSizeForClass(1));
-    for (size_t i = 1; i < kNumBins; i++) {
-      _shuffleVector[i].initialInit(arenaBegin, SizeMap::ByteSizeForClass(i));
-    }
-    d_assert(_global != nullptr);
-  }
-
-  ~ThreadLocalHeap() {
-    releaseAll();
-  }
-
-  // pthread_set_sepcific destructor
-  static void DestroyThreadLocalHeap(void *ptr);
-
-  static void InitTLH();
-
-  void releaseAll();
-
-  void *ATTRIBUTE_NEVER_INLINE CACHELINE_ALIGNED_FN smallAllocSlowpath(size_t sizeClass);
-  void *ATTRIBUTE_NEVER_INLINE CACHELINE_ALIGNED_FN smallAllocGlobalRefill(ShuffleVector &shuffleVector,
-                                                                           size_t sizeClass);
-
-  inline void *memalign(size_t alignment, size_t size) {
-    // Check for non power-of-two alignment.
-    if ((alignment == 0) || (alignment & (alignment - 1))) {
-      return nullptr;
-    }
-
-    if (size < 8) {
-      size = 8;
-    }
-
-    uint32_t sizeClass = 0;
-    const bool isSmall = SizeMap::GetSizeClass(size, &sizeClass);
-    if (alignment <= sizeof(double)) {
-      // all of our size classes are at least 8-byte aligned
-      auto ptr = this->malloc(size);
-      d_assert_msg((reinterpret_cast<uintptr_t>(ptr) % alignment) == 0, "%p(%zu) %% %zu != 0", ptr, size, alignment);
-      return ptr;
-    } else if (isSmall) {
-      const auto sizeClassBytes = SizeMap::ByteSizeForClass(sizeClass);
-      // if the alignment is for a small allocation that is less than
-      // the page size, and the size class size in bytes is a multiple
-      // of the alignment, just call malloc
-      if (sizeClassBytes <= kPageSize && alignment <= sizeClassBytes && (sizeClassBytes % alignment) == 0) {
-        auto ptr = this->malloc(size);
-        d_assert_msg((reinterpret_cast<uintptr_t>(ptr) % alignment) == 0, "%p(%zu) %% %zu != 0", ptr, size, alignment);
-        return ptr;
-      }
-    }
-
-    // fall back to page-aligned allocation
-    const size_t pageAlignment = (alignment + kPageSize - 1) / kPageSize;
-    const size_t pageCount = PageCount(size);
-    return _global->pageAlignedAlloc(pageAlignment, pageCount);
-  }
-
-  inline void *ATTRIBUTE_ALWAYS_INLINE realloc(void *oldPtr, size_t newSize) {
-    if (oldPtr == nullptr) {
-      return this->malloc(newSize);
-    }
-
-    if (newSize == 0) {
-      this->free(oldPtr);
-      return this->malloc(newSize);
-    }
-
-    size_t oldSize = getSize(oldPtr);
-
-    // the following is directly from tcmalloc, designed to avoid
-    // 'resizing ping pongs'
-    const size_t lowerBoundToGrow = oldSize + oldSize / 4ul;
-    const size_t upperBoundToShrink = oldSize / 2ul;
-
-    if (newSize > oldSize || newSize < upperBoundToShrink) {
-      void *newPtr = nullptr;
-      if (newSize > oldSize && newSize < lowerBoundToGrow) {
-        newPtr = this->malloc(lowerBoundToGrow);
-      }
-      if (newPtr == nullptr) {
-        newPtr = this->malloc(newSize);
-      }
-      if (unlikely(newPtr == nullptr)) {
-        return nullptr;
-      }
-      const size_t copySize = (oldSize < newSize) ? oldSize : newSize;
-      memcpy(newPtr, oldPtr, copySize);
-      this->free(oldPtr);
-      return newPtr;
-    } else {
-      // the current allocation is good enough
-      return oldPtr;
-    }
-  }
-
-  inline void *ATTRIBUTE_ALWAYS_INLINE calloc(size_t count, size_t size) {
-    if (unlikely(size && count > (size_t)-1 / size)) {
-      errno = ENOMEM;
-      return nullptr;
-    }
-
-    const size_t n = count * size;
-    void *ptr = this->malloc(n);
-
-    if (ptr != nullptr) {
-      memset(ptr, 0, n);
-    }
-
-    return ptr;
-  }
-
-  inline void *ATTRIBUTE_ALWAYS_INLINE cxxNew(size_t sz) {
-    void *ptr = this->malloc(sz);
-    if (unlikely(ptr == NULL && sz != 0)) {
-      throw std::bad_alloc();
-    }
-
-    return ptr;
-  }
-
-  // semiansiheap ensures we never see size == 0
-  inline void *ATTRIBUTE_ALWAYS_INLINE malloc(size_t sz) {
-    uint32_t sizeClass = 0;
-
-    // if the size isn't in our sizemap it is a large alloc
-    if (unlikely(!SizeMap::GetSizeClass(sz, &sizeClass))) {
-      return _global->malloc(sz);
-    }
-
-    ShuffleVector &shuffleVector = _shuffleVector[sizeClass];
-    if (unlikely(shuffleVector.isExhausted())) {
-      return smallAllocSlowpath(sizeClass);
-    }
-
-    return shuffleVector.malloc();
-  }
-
-  inline void ATTRIBUTE_ALWAYS_INLINE free(void *ptr) {
-    if (unlikely(ptr == nullptr))
-      return;
-
-    size_t startEpoch{0};
-    auto mh = _global->miniheapForWithEpoch(ptr, startEpoch);
-    if (likely(mh && mh->current() == _current && !mh->hasMeshed())) {
-      ShuffleVector &shuffleVector = _shuffleVector[mh->sizeClass()];
-      shuffleVector.free(mh, ptr);
-      return;
-    }
-
-    _global->freeFor(mh, ptr, startEpoch);
-  }
-
-  inline void ATTRIBUTE_ALWAYS_INLINE sizedFree(void *ptr, size_t sz) {
-    this->free(ptr);
-  }
-
-  inline size_t getSize(void *ptr) {
-    if (unlikely(ptr == nullptr))
-      return 0;
-
-    auto mh = _global->miniheapFor(ptr);
-    if (likely(mh && mh->current() == _current)) {
-      ShuffleVector &shuffleVector = _shuffleVector[mh->sizeClass()];
-      return shuffleVector.getSize();
-    }
-
-    return _global->getSize(ptr);
-  }
-
-  static ThreadLocalHeap *NewHeap(pthread_t current);
-  static ThreadLocalHeap *GetHeapIfPresent() {
-#ifdef MESH_HAVE_TLS
-    return _threadLocalHeap;
-#else
-    return _tlhInitialized ? reinterpret_cast<ThreadLocalHeap *>(pthread_getspecific(_heapKey)) : nullptr;
-#endif
-  }
-
-  static void DeleteHeap(ThreadLocalHeap *heap);
-
-  static ThreadLocalHeap *GetHeap() {
-    auto heap = GetHeapIfPresent();
-    if (unlikely(heap == nullptr)) {
-      return CreateHeapIfNecessary();
-    }
-    return heap;
-  }
-
-  static ThreadLocalHeap *ATTRIBUTE_NEVER_INLINE CreateHeapIfNecessary();
-
-protected:
-  ShuffleVector _shuffleVector[kNumBins] CACHELINE_ALIGNED;
-  // this cacheline is read-mostly (only changed when creating + destroying threads)
-  const pid_t _current CACHELINE_ALIGNED{0};
-  GlobalHeap *_global;
-  ThreadLocalHeap *_next{};  // protected by global heap lock
-  ThreadLocalHeap *_prev{};
-  const pthread_t _pthreadCurrent;
-  MWC _prng CACHELINE_ALIGNED;
-  const size_t _maxObjectSize;
-  LocalHeapStats _stats{};
-  bool _inSetSpecific{false};
-
-#ifdef MESH_HAVE_TLS
-  static __thread ThreadLocalHeap *_threadLocalHeap CACHELINE_ALIGNED ATTR_INITIAL_EXEC;
-#endif
-
-  static ThreadLocalHeap *_threadLocalHeaps;
-  static bool _tlhInitialized;
-  static pthread_key_t _heapKey;
-};
-
-}  // namespace mesh
-
-#endif  // MESH_THREAD_LOCAL_HEAP_H