@shd101wyy/yo 0.1.24 → 0.1.25

package/vendor/mimalloc/include/mimalloc/internal.h

@@ -131,6 +131,7 @@ void          _mi_strlcat(char* dest, const char* src, size_t dest_size);
 size_t        _mi_strlen(const char* s);
 size_t        _mi_strnlen(const char* s, size_t max_len);
 char*         _mi_strnstr(char* s, size_t max_len, const char* pat);
+bool          _mi_streq(const char* s, const char* t);
 bool          _mi_getenv(const char* name, char* result, size_t result_size);
 
 // "options.c"
@@ -175,10 +176,12 @@ mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id);
 mi_threadid_t _mi_thread_id(void) mi_attr_noexcept;
 size_t        _mi_thread_seq_id(void) mi_attr_noexcept;
 bool          _mi_is_heap_main(const mi_heap_t* heap);
+bool          _mi_is_theap_main(const mi_theap_t* theap);
 void          _mi_theap_guarded_init(mi_theap_t* theap);
 void          _mi_theap_options_init(mi_theap_t* theap);
 mi_theap_t*   _mi_theap_default_safe(void);             // ensure the returned theap is initialized
-
+mi_theap_t*   _mi_theap_main_safe(void);
+   
 // os.c
 void          _mi_os_init(void);                                            // called from process init
 void*         _mi_os_alloc(size_t size, mi_memid_t* memid);
@@ -284,13 +287,16 @@ size_t        _mi_bin(size_t size);                      // for stats
 
 // "theap.c"
 mi_theap_t*   _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld);
-void          _mi_theap_delete(mi_theap_t* theap);
+void          _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock);
 void          _mi_theap_default_set(mi_theap_t* theap);
 void          _mi_theap_cached_set(mi_theap_t* theap);
 void          _mi_theap_collect_retired(mi_theap_t* theap, bool force);
+void          _mi_theap_collect_abandon(mi_theap_t* theap);
 bool          _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg);
 void          _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page);
-void          _mi_theap_free(mi_theap_t* theap);
+bool          _mi_theap_free(mi_theap_t* theap, bool acquire_heap_theaps_lock, bool acquire_tld_theaps_lock);
+void          _mi_theap_incref(mi_theap_t* theap);
+void          _mi_theap_decref(mi_theap_t* theap);
 
 // "heap.c"
 void          _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page);
@@ -298,7 +304,7 @@ mi_decl_cold  mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap);  //
 mi_decl_cold  mi_theap_t* _mi_heap_theap_get_peek(const mi_heap_t* heap);     // get the theap for a heap without initializing (and return NULL in that case)
 void          _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to);  // in "arena.c"
 void          _mi_heap_destroy_pages(mi_heap_t* heap_from);                   // in "arena.c"
-
+void          _mi_heap_force_destroy(mi_heap_t* heap);                        // allow destroying the main heap
 
 // "stats.c"
 void          _mi_stats_init(void);
@@ -412,7 +418,7 @@ typedef struct mi_option_desc_s {
   Inlined definitions
 ----------------------------------------------------------- */
 #define MI_UNUSED(x)     (void)(x)
-#ifndef NDEBUG
+#if (MI_DEBUG>1)
 #define MI_UNUSED_RELEASE(x)
 #else
 #define MI_UNUSED_RELEASE(x)  MI_UNUSED(x)
@@ -428,6 +434,7 @@ typedef struct mi_option_desc_s {
 
 #define MI_INIT74(x)  MI_INIT64(x),MI_INIT8(x),x(),x()
 #define MI_INIT5(x)   MI_INIT4(x),x()
+#define MI_INIT6(x)   MI_INIT4(x),x(),x()
 
 #include <string.h>
 // initialize a local variable to zero; use memset as compilers optimize constant sized memset's
@@ -439,9 +446,8 @@ static inline bool _mi_is_power_of_two(uintptr_t x) {
 }
 
 // Is a pointer aligned?
-static inline bool _mi_is_aligned(void* p, size_t alignment) {
-  mi_assert_internal(alignment != 0);
-  return (((uintptr_t)p % alignment) == 0);
+static inline bool _mi_is_aligned(const void* p, size_t alignment) {
+  return (alignment==0 || ((uintptr_t)p % alignment) == 0);
 }
 
 // Align upwards
@@ -456,13 +462,11 @@ static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) {
   }
 }
 
-
 // Align a pointer upwards
-static inline uint8_t* _mi_align_up_ptr(void* p, size_t alignment) {
-  return (uint8_t*)_mi_align_up((uintptr_t)p, alignment);
+static inline void* _mi_align_up_ptr(const void* p, size_t alignment) {
+  return (void*)_mi_align_up((uintptr_t)p, alignment);
 }
 
-
 static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
   mi_assert_internal(alignment != 0);
   uintptr_t mask = alignment - 1;
@@ -474,7 +478,8 @@ static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) {
   }
 }
 
-static inline void* mi_align_down_ptr(void* p, size_t alignment) {
+// align a pointer downwards
+static inline void* _mi_align_down_ptr(const void* p, size_t alignment) {
   return (void*)_mi_align_down((uintptr_t)p, alignment);
 }
 
@@ -560,8 +565,13 @@ static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* tot
 extern mi_decl_hidden const mi_theap_t _mi_theap_empty;       // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_THREAD_LOCAL)
 extern mi_decl_hidden const mi_theap_t _mi_theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated
 
+
+static inline mi_heap_t* _mi_theap_heap(const mi_theap_t* theap) {
+  return mi_atomic_load_ptr_acquire(mi_heap_t,&theap->heap);
+}
+
 static inline bool mi_theap_is_initialized(const mi_theap_t* theap) {
-  return (theap != NULL && theap->heap != NULL);
+  return (theap != NULL && _mi_theap_heap(theap) != NULL);
 }
 
 static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t size) {
@@ -709,19 +719,35 @@ static inline size_t mi_page_usable_block_size(const mi_page_t* page) {
   return mi_page_block_size(page) - MI_PADDING_SIZE;
 }
 
-// This may change if we locate page info outside the page data slices
+static inline bool mi_page_meta_is_separated(const mi_page_t* page) {
+  #if MI_PAGE_META_IS_SEPARATED
+  // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_ALIGNED_FREE_SMALL
+  return (page->memid.memkind == MI_MEM_ARENA && page != _mi_align_down_ptr(page->page_start, MI_ARENA_SLICE_ALIGN));
+  #else
+  MI_UNUSED(page);
+  return false;  
+  #endif
+}
+
 static inline uint8_t* mi_page_slice_start(const mi_page_t* page) {
-  return (uint8_t*)page;
+  if (mi_page_meta_is_separated(page)) {  
+    // page meta info is at a separate location (at `arena->pages`)
+    return (uint8_t*)_mi_align_down_ptr(page->page_start, MI_ARENA_SLICE_ALIGN);
+  }
+  else {
+    // page meta info is at the start of the page slices
+    return (uint8_t*)page;
+  }
 }
 
-// This gives the offset relative to the start slice of a page. This may change if we ever
-// locate page info outside the page-data itself.
+// This gives the offset relative to the start slice of a page. 
 static inline size_t mi_page_slice_offset_of(const mi_page_t* page, size_t offset_relative_to_page_start) {
   return (page->page_start - mi_page_slice_start(page)) + offset_relative_to_page_start;
 }
 
+// Currently committed part of a page
 static inline size_t mi_page_committed(const mi_page_t* page) {
-  return (page->slice_committed == 0 ? mi_page_size(page) : page->slice_committed - (page->page_start - mi_page_slice_start(page)));
+  return (page->slice_committed == 0 ? mi_page_size(page) : page->slice_committed - mi_page_slice_offset_of(page,0));
 }
 
 // are all blocks in a page freed?
@@ -746,7 +772,7 @@ static inline bool mi_page_is_expandable(const mi_page_t* page) {
 }
 
 
-static inline bool mi_page_is_full(mi_page_t* page) {
+static inline bool mi_page_is_full(const mi_page_t* page) {
   const bool full = (page->reserved == page->used);
   mi_assert_internal(!full || page->free == NULL);
   return full;
@@ -798,9 +824,11 @@ static inline mi_page_flags_t mi_page_flags(const mi_page_t* page) {
   return (mi_page_xthread_id(page) & MI_PAGE_FLAG_MASK);
 }
 
-static inline void mi_page_flags_set(mi_page_t* page, bool set, mi_page_flags_t newflag) {
-  if (set) { mi_atomic_or_relaxed(&page->xthread_id, newflag); }
-      else { mi_atomic_and_relaxed(&page->xthread_id, ~newflag); }
+static inline bool mi_page_flags_set(mi_page_t* page, bool set, mi_page_flags_t newflag) {
+  mi_page_flags_t old;
+  if (set) { old = mi_atomic_or_relaxed(&page->xthread_id, newflag); }
+      else { old = mi_atomic_and_relaxed(&page->xthread_id, ~newflag); }
+  return ((old & newflag) == newflag);
 }
 
 static inline bool mi_page_is_in_full(const mi_page_t* page) {
@@ -808,7 +836,17 @@ static inline bool mi_page_is_in_full(const mi_page_t* page) {
 }
 
 static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) {
-  mi_page_flags_set(page, in_full, MI_PAGE_IN_FULL_QUEUE);
+  const bool was_in_full = mi_page_flags_set(page, in_full, MI_PAGE_IN_FULL_QUEUE);
+  if (was_in_full != in_full) {
+    // optimize: maintain pages_full_size to avoid visiting the full queue (issue #1220)
+    mi_theap_t* const theap = page->theap;
+    mi_assert_internal(theap!=NULL);
+    if (theap != NULL) {
+      const size_t size = page->capacity * mi_page_block_size(page);
+      if (in_full) { theap->pages_full_size += size; }
+              else { mi_assert_internal(size <= theap->pages_full_size); theap->pages_full_size -= size; }
+    }
+  }
 }
 
 static inline bool mi_page_has_interior_pointers(const mi_page_t* page) {
@@ -844,12 +882,12 @@ static inline bool mi_page_is_abandoned_mapped(const mi_page_t* page) {
 
 static inline void mi_page_set_abandoned_mapped(mi_page_t* page) {
   mi_assert_internal(mi_page_is_abandoned(page));
-  mi_atomic_or_relaxed(&page->xthread_id, MI_THREADID_ABANDONED_MAPPED);
+  mi_atomic_or_relaxed(&page->xthread_id, (mi_threadid_t)MI_THREADID_ABANDONED_MAPPED);
 }
 
 static inline void mi_page_clear_abandoned_mapped(mi_page_t* page) {
   mi_assert_internal(mi_page_is_abandoned_mapped(page));
-  mi_atomic_and_relaxed(&page->xthread_id, MI_PAGE_FLAG_MASK);
+  mi_atomic_and_relaxed(&page->xthread_id, (mi_threadid_t)MI_PAGE_FLAG_MASK);
 }
 
 
@@ -907,7 +945,7 @@ static inline bool mi_page_is_owned(const mi_page_t* page) {
 
 // get ownership; returns true if the page was not owned before.
 static inline bool mi_page_claim_ownership(mi_page_t* page) {
-  const uintptr_t old = mi_atomic_or_acq_rel(&page->xthread_free, 1);
+  const uintptr_t old = mi_atomic_or_acq_rel(&page->xthread_free, (uintptr_t)1);
   return ((old&1)==0);
 }
 
@@ -916,17 +954,23 @@ static inline bool mi_page_claim_ownership(mi_page_t* page) {
   Guarded objects
 ------------------------------------------------------------------- */
 #if MI_GUARDED
-
 // we always align guarded pointers in a block at an offset
 // the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones
 #define MI_BLOCK_TAG_ALIGNED   ((mi_encoded_t)(0))
 #define MI_BLOCK_TAG_GUARDED   (~MI_BLOCK_TAG_ALIGNED)
+#endif
 
 static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) {
+#if MI_GUARDED
   const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block;
   return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next == MI_BLOCK_TAG_GUARDED);
+#else
+  MI_UNUSED(block); MI_UNUSED(p);
+  return false;
+#endif  
 }
 
+#if MI_GUARDED
 static inline bool mi_theap_malloc_use_guarded(mi_theap_t* theap, size_t size) {
   // this code is written to result in fast assembly as it is on the hot path for allocation
   const size_t count = theap->guarded_sample_count - 1;  // if the rate was 0, this will underflow and count for a long time..
@@ -1006,7 +1050,7 @@ static inline uint32_t mi_ptr_encode_canary(const void* null, const void* p, con
 static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) {
   mi_track_mem_defined(block,sizeof(mi_block_t));
   mi_block_t* next;
-  #ifdef MI_ENCODE_FREELIST
+  #if MI_ENCODE_FREELIST
   next = (mi_block_t*)mi_ptr_decode(null, block->next, keys);
   #else
   MI_UNUSED(keys); MI_UNUSED(null);
@@ -1018,7 +1062,7 @@ static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* bl
 
 static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) {
   mi_track_mem_undefined(block,sizeof(mi_block_t));
-  #ifdef MI_ENCODE_FREELIST
+  #if MI_ENCODE_FREELIST
   block->next = mi_ptr_encode(null, next, keys);
   #else
   MI_UNUSED(keys); MI_UNUSED(null);
@@ -1028,7 +1072,7 @@ static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const
 }
 
 static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) {
-  #ifdef MI_ENCODE_FREELIST
+  #if MI_ENCODE_FREELIST
   mi_block_t* next = mi_block_nextx(page,block,page->keys);
   // check for free list corruption: is `next` at least in the same page?
   // TODO: check if `next` is `page->block_size` aligned?
@@ -1044,7 +1088,7 @@ static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t*
 }
 
 static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, const mi_block_t* next) {
-  #ifdef MI_ENCODE_FREELIST
+  #if MI_ENCODE_FREELIST
   mi_block_set_nextx(page,block,next, page->keys);
   #else
   MI_UNUSED(page);

package/vendor/mimalloc/include/mimalloc/prim.h

@@ -126,6 +126,9 @@ void _mi_prim_thread_associate_default_theap(mi_theap_t* theap);
 // Is this thread part of a thread pool?
 bool _mi_prim_thread_is_in_threadpool(void);
 
+// Yield to other threads. Should be similar to `sleep(0)`. 
+// Is called only in rare situations and does not have to be lightning fast.
+void _mi_prim_thread_yield(void);
 
 //-------------------------------------------------------------------
 // Access to TLS (thread local storage) slots.
@@ -253,14 +256,16 @@ extern mi_decl_hidden bool _mi_process_is_initialized;                // has mi_
 // but unfortunately, it seems we cannot test for this reliably at this time (see issue #883)
 // Nevertheless, it seems needed on older graviton platforms (see issue #851).
 // For now, we only enable this for specific platforms.
-#if !defined(__APPLE__)  /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \
-    && !defined(__CYGWIN__) \
-    && !defined(MI_LIBC_MUSL) \
-    && (!defined(__clang_major__) || __clang_major__ >= 14)  /* older clang versions emit bad code; fall back to using the TLS slot (<https://lore.kernel.org/linux-arm-kernel/202110280952.352F66D8@keescook/T/>) */
-  #if    (defined(__GNUC__) && (__GNUC__ >= 7)  && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \
-      || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \
-      || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__)))
-    #define MI_USE_BUILTIN_THREAD_POINTER  1
+#if !defined(MI_USE_BUILTIN_THREAD_POINTER)   /* allow user override */
+  #if !defined(__APPLE__)  /* on apple (M1) the wrong register is read (tpidr_el0 instead of tpidrro_el0) so fall back to TLS slot assembly (<https://github.com/microsoft/mimalloc/issues/343#issuecomment-763272369>)*/ \
+      && !defined(__CYGWIN__) \
+      && !defined(MI_LIBC_MUSL) \
+      && (!defined(__clang_major__) || __clang_major__ >= 14)  /* older clang versions emit bad code; fall back to using the TLS slot (<https://lore.kernel.org/linux-arm-kernel/202110280952.352F66D8@keescook/T/>) */
+    #if    (defined(__GNUC__) && (__GNUC__ >= 7)  && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \
+        || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \
+        || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__)))
+      #define MI_USE_BUILTIN_THREAD_POINTER  1
+    #endif
   #endif
 #endif
 
@@ -276,8 +281,10 @@ static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
 // Get a unique id for the current thread.
 #if defined(MI_PRIM_THREAD_ID)
 
-static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept {
-  return MI_PRIM_THREAD_ID();  // used for example by CPython for a free threaded build (see python/cpython#115488)
+static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept {
+  const mi_threadid_t tid = MI_PRIM_THREAD_ID();  // used for example by CPython for a free threaded build (see python/cpython#115488)
+  mi_assert_internal( (tid & 0x03) == 0 );        // mimalloc reserves the bottom 2 bits
+  return tid;
 }
 
 #elif defined(_WIN32)
@@ -363,14 +370,14 @@ static inline mi_theap_t* _mi_theap_cached(void);
 
 #if defined(_WIN32)
   #define MI_TLS_MODEL_DYNAMIC_WIN32        1    
-#elif defined(__APPLE__)  // macOS
+#elif defined(__APPLE__) && MI_HAS_TLS_SLOT && !defined(__POWERPC__)  // macOS on arm64 or x64
   // #define MI_TLS_MODEL_DYNAMIC_PTHREADS  1    // also works but a bit slower
   #define MI_TLS_MODEL_FIXED_SLOT           1
   #define MI_TLS_MODEL_FIXED_SLOT_DEFAULT   108  // seems unused. @apple: it would be great to get 2 official slots for custom allocators :-)
   #define MI_TLS_MODEL_FIXED_SLOT_CACHED    109
   // we used before __PTK_FRAMEWORK_OLDGC_KEY9 (89) but that seems used now.
   // see <https://github.com/rweichler/substrate/blob/master/include/pthread_machdep.h>
-#elif defined(__OpenBSD__) || defined(__ANDROID__)
+#elif defined(__APPLE__) || defined(__OpenBSD__) || defined(__ANDROID__)
   #define MI_TLS_MODEL_DYNAMIC_PTHREADS     1
   // #define MI_TLS_MODEL_DYNAMIC_PTHREADS_DEFAULT_ENTRY_IS_NULL  1
 #else
@@ -497,11 +504,11 @@ static inline bool _mi_thread_is_initialized(void) {
 // Get (and possible create) the theap belonging to a heap
 // We cache the last accessed theap in `_mi_theap_cached` for better performance.
 static inline mi_theap_t* _mi_heap_theap(const mi_heap_t* heap) {
-  mi_theap_t* theap = _mi_theap_cached();
+  mi_theap_t* theap = _mi_theap_cached();  
   #if MI_THEAP_INITASNULL
-  if mi_likely(theap!=NULL && theap->heap==heap) return theap;
+  if mi_likely(theap!=NULL && _mi_theap_heap(theap)==heap) return theap;
   #else
-  if mi_likely(theap->heap==heap) return theap;
+  if mi_likely(_mi_theap_heap(theap)==heap) return theap;
   #endif
   return _mi_heap_theap_get_or_init(heap);
 }
@@ -510,14 +517,14 @@ static inline mi_theap_t* _mi_heap_theap(const mi_heap_t* heap) {
 static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) {
   mi_theap_t* theap = _mi_theap_cached();
   #if MI_THEAP_INITASNULL
-  if mi_unlikely(theap==NULL || theap->heap!=heap)
+  if mi_unlikely(theap==NULL || _mi_theap_heap(theap)!=heap)
   #else
-  if mi_unlikely(theap->heap!=heap)
+  if mi_unlikely(_mi_theap_heap(theap)!=heap)
   #endif
   {
     theap = _mi_heap_theap_get_peek(heap);  // don't update the cache on a query (?)
   }
-  mi_assert(theap==NULL || theap->heap==heap);
+  mi_assert(theap==NULL || _mi_theap_heap(theap)==heap);
   return theap;
 }
 

package/vendor/mimalloc/include/mimalloc/track.h

@@ -84,7 +84,8 @@ defined, undefined, or not accessible at all:
 
 #include "../src/prim/windows/etw.h"
 
-#define mi_track_init()                           EventRegistermicrosoft_windows_mimalloc();
+#define mi_track_init()                           EventRegistermicrosoft_windows_mimalloc()
+#define mi_track_done()                           EventUnregistermicrosoft_windows_mimalloc()
 #define mi_track_malloc_size(p,reqsize,size,zero) EventWriteETW_MI_ALLOC((UINT64)(p), size)
 #define mi_track_free_size(p,size)                EventWriteETW_MI_FREE((UINT64)(p), size)
 
@@ -104,7 +105,7 @@ defined, undefined, or not accessible at all:
 // Utility definitions
 
 #ifndef mi_track_resize
-#define mi_track_resize(p,oldsize,newsize)      mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false)
+#define mi_track_resize(p,oldsize,newsize)      do{ mi_track_free_size(p,oldsize); mi_track_malloc(p,newsize,false); } while(0)
 #endif
 
 #ifndef mi_track_align
@@ -115,6 +116,10 @@ defined, undefined, or not accessible at all:
 #define mi_track_init()
 #endif
 
+#ifndef mi_track_done
+#define mi_track_done()
+#endif
+
 #ifndef mi_track_mem_defined
 #define mi_track_mem_defined(p,size)
 #endif

package/vendor/mimalloc/include/mimalloc/types.h

@@ -54,17 +54,12 @@ terms of the MIT license. A copy of the license can be found in the file
 // Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance).
 // #define MI_STAT 1
 
-// Define MI_SECURE to enable security mitigations. Level 1 has minimal performance impact,
-// but protects most metadata with guard pages:
-//   #define MI_SECURE 1  // guard page around metadata; check pointer validity on free
-//
-// Level 2 has more performance impact but protect well against various buffer overflows
-// by surrounding all mimalloc pages with guard pages:
-//   #define MI_SECURE 2  // guard page around each mimalloc page (can fragment VMA's with large theaps..)
-//
-// The next two levels can have more performance cost:
-//   #define MI_SECURE 3  // randomize allocations, encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free)
-//   #define MI_SECURE 4  // checks for double free. (may be more expensive)
+// Define MI_SECURE to enable security mitigations
+// #define MI_SECURE 1  // guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption
+// #define MI_SECURE 2  // randomize relative allocation addresses (within mimalloc pages)
+// #define MI_SECURE 3  // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free)
+// #define MI_SECURE 4  // checks for double free (may be more expensive) (`-DMI_SECURE=ON`)
+// #define MI_SECURE 5  // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`)
 
 #if !defined(MI_SECURE)
 #define MI_SECURE 0
@@ -91,11 +86,9 @@ terms of the MIT license. A copy of the license can be found in the file
 #endif
 #endif
 
-// Use guard pages behind objects of a certain size (set by the MIMALLOC_DEBUG_GUARDED_MIN/MAX options)
-// Padding should be disabled when using guard pages
-// #define MI_GUARDED 1
-#if MI_GUARDED
-#define MI_PADDING  0
+// Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options)
+#if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_PAGE_META_ALIGNED_FREE_SMALL 
+#define MI_GUARDED  1
 #endif
 
 // Reserve extra padding at the end of each block to be more resilient against theap block overflows.
@@ -123,6 +116,40 @@ terms of the MIT license. A copy of the license can be found in the file
 #define MI_ENABLE_LARGE_PAGES  1
 #endif
 
+// Place page meta info at the start of the page area or keep it separate? 
+// Separate keeps the page info at the arena start (default) which is more secure 
+// and reduces wasted space due to alignment and block sizes. 
+// (but also reserves more memory up front (about 2MiB per GiB))
+#if !defined(MI_PAGE_META_IS_SEPARATED)
+#if MI_PAGE_MAP_FLAT
+#define MI_PAGE_META_IS_SEPARATED    0
+#else
+#define MI_PAGE_META_IS_SEPARATED    1
+#endif
+#endif
+
+// We can choose to only put page info of small pages at the start of the page area.
+// This can be used to have a slightly faster `mi_free_small` function for specialized
+// cases (like language runtime systems).
+#if !defined(MI_PAGE_META_ALIGNED_FREE_SMALL)
+#define MI_PAGE_META_ALIGNED_FREE_SMALL   0
+#endif
+
+// Configuration checks
+#if !MI_PAGE_META_IS_SEPARATED && MI_SECURE
+#error "secure mode should use separated page infos"
+#endif
+#if MI_PAGE_META_ALIGNED_FREE_SMALL && MI_SECURE
+#error "secure mode cannot use MI_PAGE_META_ALIGNED_FREE_SMALL"
+#endif
+#if MI_PAGE_META_IS_SEPARATED && MI_PAGE_MAP_FLAT
+#error "cannot have a flat page map with separated page infos"
+#endif
+#if MI_DEBUG && NDEBUG
+#warning "mimalloc assertions enabled in a release build"
+#endif
+
+
 // --------------------------------------------------------------
 // Sizes of internal data-structures
 // (comments specify sizes on 64-bit, usually 32-bit is halved)
@@ -132,7 +159,7 @@ terms of the MIT license. A copy of the license can be found in the file
 #ifndef MI_ARENA_SLICE_SHIFT
   #ifdef  MI_SMALL_PAGE_SHIFT   // backward compatibility
   #define MI_ARENA_SLICE_SHIFT              MI_SMALL_PAGE_SHIFT
-  #elif MI_SECURE && __APPLE__ && MI_ARCH_ARM64
+  #elif MI_SECURE>=5 && __APPLE__ && MI_ARCH_ARM64
   #define MI_ARENA_SLICE_SHIFT              (17)                        // 128 KiB to not waste too much due to 16 KiB guard pages
   #else
   #define MI_ARENA_SLICE_SHIFT              (13 + MI_SIZE_SHIFT)        // 64 KiB (32 KiB on 32-bit)
@@ -313,7 +340,7 @@ typedef size_t mi_page_flags_t;
 // There are two special threadid's: 0 for pages that are abandoned (and not in a theap queue),
 // and 4 for abandoned & mapped threads -- abandoned-mapped pages are abandoned but also mapped
 // in an arena (in `mi_heap_t.arena_pages.pages_abandoned`) so these can be quickly found for reuse.
-// Abondoning partially used pages allows for sharing of this memory between threads (in particular if threads are blocked)
+// Abandoning partially used pages allows for sharing of this memory between threads (in particular if threads are blocked)
 #define MI_THREADID_ABANDONED           MI_ZU(0)
 #define MI_THREADID_ABANDONED_MAPPED    (MI_PAGE_FLAG_MASK + 1)
 
@@ -392,12 +419,6 @@ typedef struct mi_page_s {
 #define MI_PAGE_OSPAGE_BLOCK_ALIGN2       (4*MI_KiB)                // also aligns any multiple of this size to avoid TLB misses.
 #define MI_PAGE_MAX_OVERALLOC_ALIGN       MI_ARENA_SLICE_SIZE       // (64 KiB) limit for which we overallocate in arena pages, beyond this use OS allocation
 
-#if (MI_ENCODE_FREELIST || MI_PADDING) && MI_SIZE_SIZE == 8
-#define MI_PAGE_INFO_SIZE                 ((MI_INTPTR_SHIFT+2)*32)  // 160    >= sizeof(mi_page_t)
-#else
-#define MI_PAGE_INFO_SIZE                 ((MI_INTPTR_SHIFT+1)*32)  // 128/96 >= sizeof(mi_page_t)
-#endif
-
 // The max object sizes are intended to not waste more than ~ 12.5% internally over the page sizes.
 #define MI_SMALL_MAX_OBJ_SIZE             ((MI_SMALL_PAGE_SIZE-MI_PAGE_OSPAGE_BLOCK_ALIGN2)/6)   // = 10 KiB
 #if MI_ENABLE_LARGE_PAGES
@@ -423,7 +444,7 @@ typedef enum mi_page_kind_e {
   MI_PAGE_MEDIUM,     // medium blocks go into 512KiB pages
   MI_PAGE_LARGE,      // larger blocks go into 4MiB pages (if `MI_ENABLE_LARGE_PAGES==1`)
   MI_PAGE_SINGLETON   // page containing a single block.
-                      // used for blocks `> MI_LARGE_MAX_OBJ_SIZE` or an aligment `> MI_PAGE_MAX_OVERALLOC_ALIGN`.
+                      // used for blocks `> MI_LARGE_MAX_OBJ_SIZE` or an alignment `> MI_PAGE_MAX_OVERALLOC_ALIGN`.
 } mi_page_kind_t;
 
 
@@ -482,21 +503,23 @@ typedef struct mi_padding_s {
 // A thread-local heap ("theap") owns a set of thread-local pages.
 struct mi_theap_s {
   mi_tld_t*             tld;                                 // thread-local data
-  mi_heap_t*            heap;                                // the heap this theap belongs to.
+  _Atomic(mi_heap_t*)   heap;                                // the heap this theap belongs to.
+  _Atomic(size_t)       refcount;                            // reference count
   unsigned long long    heartbeat;                           // monotonic heartbeat count
   uintptr_t             cookie;                              // random cookie to verify pointers (see `_mi_ptr_cookie`)
   mi_random_ctx_t       random;                              // random number context used for secure allocation
   size_t                page_count;                          // total number of pages in the `pages` queues.
   size_t                page_retired_min;                    // smallest retired index (retired pages are fully free, but still in the page queues)
   size_t                page_retired_max;                    // largest retired index into the `pages` array.
+  size_t                pages_full_size;                     // optimization: total size of blocks in the pages of the full queue (issue #1220)
   long                  generic_count;                       // how often is `_mi_malloc_generic` called?
   long                  generic_collect_count;               // how often is `_mi_malloc_generic` called without collecting?
 
   mi_theap_t*           tnext;                               // list of theaps in this thread
-  mi_theap_t*           tprev;
+  mi_theap_t*           tprev;  
   mi_theap_t*           hnext;                               // list of theaps of the owning `heap`
   mi_theap_t*           hprev;
-
+  
   long                  page_full_retain;                    // how many full pages can be retained per queue (before abandoning them)
   bool                  allow_page_reclaim;                  // `true` if this theap should not reclaim abandoned pages
   bool                  allow_page_abandon;                  // `true` if this theap can abandon pages to reduce memory footprint
@@ -604,6 +627,7 @@ struct mi_tld_s {
   int                   numa_node;            // thread preferred numa node
   mi_subproc_t*         subproc;              // sub-process this thread belongs to.
   mi_theap_t*           theaps;               // list of theaps in this thread (so we can abandon all when the thread terminates)
+  mi_lock_t             theaps_lock;          // lock as the theaps list is sometimes accessed from another thread (on `mi_heap_free`)
   bool                  recurse;              // true if deferred was called; used to prevent infinite recursion.
   bool                  is_in_threadpool;     // true if this thread is part of a threadpool (and can run arbitrary tasks)
   mi_memid_t            memid;                // provenance of the tld memory itself (meta or OS)
@@ -632,7 +656,7 @@ typedef struct mi_bbitmap_s mi_bbitmap_t;   // atomic binned bitmap (defined in
 typedef struct mi_arena_pages_s {
   mi_bitmap_t* pages;                // all registered pages (abandoned and owned)
   mi_bitmap_t* pages_abandoned[MI_ARENA_BIN_COUNT];  // abandoned pages per size bin (a set bit means the start of the page)
-  // followed by the bitmaps (whose sizes depend on the arena size)
+  // followed by the bitmaps (whose siz`es depend on the arena size)
 } mi_arena_pages_t;
 
 
@@ -651,10 +675,14 @@ typedef struct mi_arena_s {
   mi_commit_fun_t*    commit_fun;           // custom commit/decommit memory
   void*               commit_fun_arg;       // user argument for a custom commit function
 
+  size_t              total_size;           // for (user given) memory more than MI_ARENA_MAX_SIZE, we use N arena's to cover it. The first (parent) has the total size (and the other sub-arena's 0).
+  mi_arena_t*         parent;               // if this is a sub arena, this points to the first one in the memory area.
+
   mi_bbitmap_t*       slices_free;          // is the slice free? (a binned bitmap with size classes)
   mi_bitmap_t*        slices_committed;     // is the slice committed? (i.e. accessible)
   mi_bitmap_t*        slices_dirty;         // is the slice potentially non-zero?
   mi_bitmap_t*        slices_purge;         // slices that can be purged
+  mi_page_t*          pages_meta;           // pre-allocated `slice_count` page meta info -- only used if `MI_PAGE_META_IS_SEPARATED!=0`
   mi_arena_pages_t    pages_main;           // arena page bitmaps for the main heap are allocated up front as well
 
   // followed by the bitmaps (whose sizes depend on the arena size)

package/vendor/mimalloc/include/mimalloc-override.h

@@ -19,7 +19,7 @@ not accidentally mix pointers from different allocators).
 
 // Standard C allocation
 #define malloc(n)               mi_malloc(n)
-#define calloc(n,c)             mi_calloc(n,c)
+#define calloc(c,n)             mi_calloc(c,n)
 #define realloc(p,n)            mi_realloc(p,n)
 #define free(p)                 mi_free(p)
 
@@ -30,26 +30,26 @@ not accidentally mix pointers from different allocators).
 // Microsoft extensions
 #define _expand(p,n)            mi_expand(p,n)
 #define _msize(p)               mi_usable_size(p)
-#define _recalloc(p,n,c)        mi_recalloc(p,n,c)
+#define _recalloc(p,c,n)        mi_recalloc(p,c,n)
 
 #define _strdup(s)              mi_strdup(s)
 #define _strndup(s,n)           mi_strndup(s,n)
-#define _wcsdup(s)              (wchar_t*)mi_wcsdup((const unsigned short*)(s))
+#define _wcsdup(s)              mi_wcsdup(s)
 #define _mbsdup(s)              mi_mbsdup(s)
-#define _dupenv_s(b,n,v)        mi_dupenv_s(b,n,v)
-#define _wdupenv_s(b,n,v)       mi_wdupenv_s((unsigned short*)(b),n,(const unsigned short*)(v))
+#define _dupenv_s(buf,n,nm)     mi_dupenv_s(buf,n,nm)
+#define _wdupenv_s(buf,n,nm)    mi_wdupenv_s(buf,n,nm)
 
 // Various Posix and Unix variants
 #define reallocf(p,n)           mi_reallocf(p,n)
 #define malloc_size(p)          mi_usable_size(p)
 #define malloc_usable_size(p)   mi_usable_size(p)
-#define malloc_good_size(sz)    mi_malloc_good_size(sz)
+#define malloc_good_size(n)     mi_malloc_good_size(n)
 #define cfree(p)                mi_free(p)
 
 #define valloc(n)               mi_valloc(n)
 #define pvalloc(n)              mi_pvalloc(n)
-#define reallocarray(p,s,n)     mi_reallocarray(p,s,n)
-#define reallocarr(p,s,n)       mi_reallocarr(p,s,n)
+#define reallocarray(p,c,n)     mi_reallocarray(p,c,n)
+#define reallocarr(ptrp,c,n)    mi_reallocarr(ptrp,c,n)
 #define memalign(a,n)           mi_memalign(a,n)
 #define aligned_alloc(a,n)      mi_aligned_alloc(a,n)
 #define posix_memalign(p,a,n)   mi_posix_memalign(p,a,n)
@@ -58,11 +58,11 @@ not accidentally mix pointers from different allocators).
 // Microsoft aligned variants
 #define _aligned_malloc(n,a)                  mi_malloc_aligned(n,a)
 #define _aligned_realloc(p,n,a)               mi_realloc_aligned(p,n,a)
-#define _aligned_recalloc(p,s,n,a)            mi_aligned_recalloc(p,s,n,a)
+#define _aligned_recalloc(p,c,n,a)            mi_aligned_recalloc(p,c,n,a)
 #define _aligned_msize(p,a,o)                 mi_usable_size(p)
 #define _aligned_free(p)                      mi_free(p)
 #define _aligned_offset_malloc(n,a,o)         mi_malloc_aligned_at(n,a,o)
 #define _aligned_offset_realloc(p,n,a,o)      mi_realloc_aligned_at(p,n,a,o)
-#define _aligned_offset_recalloc(p,s,n,a,o)   mi_recalloc_aligned_at(p,s,n,a,o)
+#define _aligned_offset_recalloc(p,c,n,a,o)   mi_recalloc_aligned_at(p,c,n,a,o)
 
 #endif // MIMALLOC_OVERRIDE_H