@shd101wyy/yo 0.1.23 → 0.1.25

package/vendor/mimalloc/src/bitmap.c

@@ -12,6 +12,7 @@ Concurrent bitmap that can set/reset sequences of bits atomically
 #include "mimalloc.h"
 #include "mimalloc/internal.h"
 #include "mimalloc/bits.h"
+#include "mimalloc/prim.h"  // _mi_prim_thread_yield
 #include "bitmap.h"
 
 #ifndef MI_OPT_SIMD
@@ -119,7 +120,7 @@ static inline void mi_bfield_atomic_clear_once_set(_Atomic(mi_bfield_t)*b, size_
         mi_subproc_stat_counter_increase(_mi_subproc(), pages_unabandon_busy_wait, 1);
       }
       while ((old&mask)==0) { // busy wait
-        mi_atomic_yield();
+        _mi_prim_thread_yield(); 
         old = mi_atomic_load_acquire(b);
       }
     }
@@ -220,13 +221,13 @@ static inline bool mi_bfield_atomic_try_clearX(_Atomic(mi_bfield_t)*b, bool* all
 
 // Check if a bit is set
 static inline bool mi_bfield_atomic_is_set(const _Atomic(mi_bfield_t)*b, const size_t idx) {
-  const mi_bfield_t x = mi_atomic_load_relaxed(b);
+  const mi_bfield_t x = mi_atomic_load_acquire(b);
   return ((x & mi_bfield_mask(1,idx)) != 0);
 }
 
 // Check if a bit is clear
 static inline bool mi_bfield_atomic_is_clear(const _Atomic(mi_bfield_t)*b, const size_t idx) {
-  const mi_bfield_t x = mi_atomic_load_relaxed(b);
+  const mi_bfield_t x = mi_atomic_load_acquire(b);
   return ((x & mi_bfield_mask(1, idx)) == 0);
 }
 
@@ -239,14 +240,14 @@ static inline bool mi_bfield_atomic_is_xset(mi_xset_t set, const _Atomic(mi_bfie
 // Check if all bits corresponding to a mask are set.
 static inline bool mi_bfield_atomic_is_set_mask(const _Atomic(mi_bfield_t)* b, mi_bfield_t mask) {
   mi_assert_internal(mask != 0);
-  const mi_bfield_t x = mi_atomic_load_relaxed(b);
+  const mi_bfield_t x = mi_atomic_load_acquire(b);
   return ((x & mask) == mask);
 }
 
 // Check if all bits corresponding to a mask are clear.
 static inline bool mi_bfield_atomic_is_clear_mask(const _Atomic(mi_bfield_t)* b, mi_bfield_t mask) {
   mi_assert_internal(mask != 0);
-  const mi_bfield_t x = mi_atomic_load_relaxed(b);
+  const mi_bfield_t x = mi_atomic_load_acquire(b);
   return ((x & mask) == 0);
 }
 
@@ -259,7 +260,7 @@ static inline bool mi_bfield_atomic_is_xset_mask(mi_xset_t set, const _Atomic(mi
 
 // Count bits in a mask
 static inline size_t mi_bfield_atomic_popcount_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask) {
-  const mi_bfield_t x = mi_atomic_load_relaxed(b);
+  const mi_bfield_t x = mi_atomic_load_acquire(b);
   return mi_bfield_popcount(x & mask);
 }
 
@@ -962,6 +963,19 @@ static bool mi_bchunk_bsr(mi_bchunk_t* chunk, size_t* pidx) {
   return false;
 }
 
+static bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx) {
+  for (size_t i = MI_BCHUNK_FIELDS; i > 0; ) {
+    i--;
+    mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]);
+    size_t idx;
+    if (mi_bsr(~b, &idx)) {
+      *pidx = (i*MI_BFIELD_BITS) + idx;
+      return true;
+    }
+  }
+  return false;
+}
+
 static size_t mi_bchunk_popcount(mi_bchunk_t* chunk) {
   size_t popcount = 0;
   for (size_t i = 0; i < MI_BCHUNK_FIELDS; i++) {
@@ -1059,7 +1073,6 @@ static void mi_bchunks_unsafe_setN(mi_bchunk_t* chunks, mi_bchunkmap_t* cmap, si
   // last chunk
   if (n > 0) {
     mi_assert_internal(n < MI_BCHUNK_BITS);
-    mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS);
     mi_bchunk_setN(&chunks[chunk_idx], 0, n, NULL);
   }
 }
@@ -1428,22 +1441,28 @@ bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* vis
       mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx];
       for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) {
         const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS);
-        mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], 0);
-#if MI_DEBUG > 1
+        mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0);
+        #if MI_DEBUG > 1
         const size_t bpopcount = mi_popcount(b);
         size_t rngcount = 0;
-#endif
+        #endif
         size_t bidx;
         while (mi_bfield_find_least_bit(b, &bidx)) {
           size_t rng = mi_ctz(~(b>>bidx)); // all the set bits from bidx
-#if MI_DEBUG > 1
+          #if MI_DEBUG > 1
           rngcount += rng;
-#endif
+          #endif
           const size_t idx = base_idx + bidx;
           mi_assert_internal(rng>=1 && rng<=MI_BFIELD_BITS);
           mi_assert_internal((idx % MI_BFIELD_BITS) + rng <= MI_BFIELD_BITS);
           mi_assert_internal((idx / MI_BCHUNK_BITS) < mi_bitmap_chunk_count(bitmap));
-          if (!visit(idx, rng, arena, arg)) return false;
+          if (!visit(idx, rng, arena, arg)) {
+            // break early: reset the non-visited bits
+            if (b!=0) {
+              mi_atomic_or_relaxed(&chunk->bfields[j], b);
+            }
+            return false;
+          }
           // clear rng bits in b
           b = b & ~mi_bfield_mask(rng, bidx);
         }
@@ -1481,18 +1500,23 @@ bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_fo
       mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx];
       for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) {
         const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS);
-        mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], 0);                // atomic clear
+        mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0);   // atomic clear
         mi_bfield_t skipped = 0;                                                          // but track which bits we skip so we can restore them
         for(size_t shift = 0; rngslices + shift <= MI_BFIELD_BITS; shift += rngslices) {  // per `rngslices` to keep alignment
           const mi_bfield_t rngmask = mi_bfield_mask(rngslices, shift);
           if ((b & rngmask) == rngmask) {
             const size_t idx = base_idx + shift;
             if (!visit(idx, rngslices, arena, arg)) {
-              // break early
-              if (skipped != 0) {
-                mi_atomic_or_relaxed(&chunk->bfields[j], skipped);
-                return false;
+              // break early: restore non-visited entries
+              mi_bfield_t notyet_visited = 0;
+              if (shift + rngslices < MI_BFIELD_BITS) {
+                notyet_visited = (b & (~(mi_bfield_t)0 << (shift + rngslices)));
               }
+              mi_assert_internal((notyet_visited & skipped) == 0);
+              if ((notyet_visited | skipped) != 0) {
+                mi_atomic_or_relaxed(&chunk->bfields[j], notyet_visited | skipped);
+              }
+              return false;
             }
           }
           else {
@@ -1547,6 +1571,32 @@ void mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) {
   mi_bchunks_unsafe_setN(&bbitmap->chunks[0], &bbitmap->chunkmap, idx, n);
 }
 
+bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx) {
+  const size_t chunk_count = mi_bbitmap_chunk_count(bbitmap);
+  const size_t chunkmap_max = _mi_divide_up(chunk_count, MI_BFIELD_BITS);
+  size_t skip_at_top = chunk_count % MI_BFIELD_BITS;
+  for (size_t i = chunkmap_max; i > 0; ) {
+    i--;
+    mi_bfield_t cmap = mi_atomic_load_relaxed(&bbitmap->chunkmap.bfields[i]);
+    size_t cmap_idx;
+    // don't consider top 0 bits; set those to 1 here
+    if (skip_at_top > 0) {
+      const size_t mask_top = (~mi_bfield_zero()) << (MI_BFIELD_BITS - skip_at_top);
+      skip_at_top = 0;   // only for the first iteration
+      cmap |= mask_top;
+    }
+    if (mi_bsr(~cmap, &cmap_idx)) {
+      // highest chunk
+      const size_t chunk_idx = i*MI_BFIELD_BITS + cmap_idx;
+      size_t cidx;
+      if (mi_bchunk_bsr_inv(&bbitmap->chunks[chunk_idx], &cidx)) {
+        *idx = (chunk_idx * MI_BCHUNK_BITS) + cidx;
+        return true;
+      }
+    }
+  }
+  return false;
+}
 
 
 /* --------------------------------------------------------------------------------

package/vendor/mimalloc/src/bitmap.h

@@ -281,6 +281,10 @@ mi_chunkbin_t mi_bbitmap_debug_get_bin(const mi_bchunk_t* chunkmap_bins, size_t
 
 size_t mi_bbitmap_size(size_t bit_count, size_t* chunk_count);
 
+// If a bit is clear in the bitmap, return `true` and set `idx` to the index of the highest bit that was clear.
+// Otherwise return `false` (and `*idx` is undefined).
+// Used for debug output.
+bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx);
 
 // Initialize a bitmap to all clear; avoid a mem_zero if `already_zero` is true
 // returns the size of the bitmap.
@@ -319,13 +323,13 @@ bool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n);
 bool mi_bbitmap_try_find_and_clear(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx);  // 1-bit
 bool mi_bbitmap_try_find_and_clear8(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // 8-bits
 // bool mi_bbitmap_try_find_and_clearX(mi_bbitmap_t* bbitmap, size_t tseq, size_t* pidx); // MI_BFIELD_BITS
-bool mi_bbitmap_try_find_and_clearNX(mi_bbitmap_t* bbitmap, size_t n, size_t tseq, size_t* pidx); // < MI_BFIELD_BITS
-bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t n, size_t tseq, size_t* pidx); // > MI_BFIELD_BITS <= MI_BCHUNK_BITS
-bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t n, size_t tseq, size_t* pidx); // > MI_BCHUNK_BITS
+bool mi_bbitmap_try_find_and_clearNX(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // < MI_BFIELD_BITS
+bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // > MI_BFIELD_BITS <= MI_BCHUNK_BITS
+bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx); // > MI_BCHUNK_BITS
 
 // Find a sequence of `n` bits in the bbitmap with all bits set, and try to atomically clear all.
 // Returns true on success, and in that case sets the index: `0 <= *pidx <= MI_BITMAP_MAX_BITS-n`.
-mi_decl_nodiscard static inline bool mi_bbitmap_try_find_and_clearN(mi_bbitmap_t* bbitmap, size_t n, size_t tseq, size_t* pidx) {
+mi_decl_nodiscard static inline bool mi_bbitmap_try_find_and_clearN(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx) {
   if (n==1) return mi_bbitmap_try_find_and_clear(bbitmap, tseq, pidx);               // small pages
   if (n==8) return mi_bbitmap_try_find_and_clear8(bbitmap, tseq, pidx);              // medium pages
   // if (n==MI_BFIELD_BITS) return mi_bbitmap_try_find_and_clearX(bbitmap, tseq, pidx); // large pages

package/vendor/mimalloc/src/free.c

@@ -15,7 +15,7 @@ terms of the MIT license. A copy of the license can be found in the file
 // forward declarations
 static void   mi_check_padding(const mi_page_t* page, const mi_block_t* block);
 static bool   mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block);
-static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block);
+static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded);
 static void   mi_stat_free(const mi_page_t* page, const mi_block_t* block);
 
 
@@ -25,16 +25,17 @@ static void   mi_stat_free(const mi_page_t* page, const mi_block_t* block);
 
 // regular free of a (thread local) block pointer
 // fast path written carefully to prevent spilling on the stack
-static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool track_stats, bool check_full)
+static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool track_stats, bool check_full)
 {
-  // checks
+  MI_UNUSED(was_guarded);
+  // checks  
   if mi_unlikely(mi_check_is_double_free(page, block)) return;
-  mi_check_padding(page, block);
+  if (!was_guarded) { mi_check_padding(page, block); }
   if (track_stats) { mi_stat_free(page, block); }
-  #if (MI_DEBUG>0) && !MI_TRACK_ENABLED  && !MI_TSAN && !MI_GUARDED
+  #if (MI_DEBUG>0) && !MI_TRACK_ENABLED  && !MI_TSAN
   memset(block, MI_DEBUG_FREED, mi_page_block_size(page));
   #endif
-  if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block)); } // faster then mi_usable_size as we already know the page and that p is unaligned
+  if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); } // faster then mi_usable_size as we already know the page and that p is unaligned
 
   // actual free: push on the local free list
   mi_block_set_next(page, block, page->local_free);
@@ -53,17 +54,20 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool
 static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept;
 
 // Free a block multi-threaded
-static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block) mi_attr_noexcept
+static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded) mi_attr_noexcept
 {
+  MI_UNUSED(was_guarded);
   // adjust stats (after padding check and potentially recursive `mi_free` above)
   mi_stat_free(page, block);    // stat_free may access the padding
-  mi_track_free_size(block, mi_page_usable_size_of(page, block));
+  mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded));
 
   // _mi_padding_shrink(page, block, sizeof(mi_block_t));
 #if (MI_DEBUG>0) && !MI_TRACK_ENABLED  && !MI_TSAN       // note: when tracking, cannot use mi_usable_size with multi-threading
-  size_t dbgsize = mi_usable_size(block);
-  if (dbgsize > MI_MiB) { dbgsize = MI_MiB; }
-  _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize);
+  if (!was_guarded) {
+    size_t dbgsize = mi_usable_size(block);
+    if (dbgsize > MI_MiB) { dbgsize = MI_MiB; }
+    _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize);
+  }
 #endif
 
   // push atomically on the page thread free list
@@ -99,12 +103,19 @@ mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) {
 // forward declaration for a MI_GUARDED build
 #if MI_GUARDED
 static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration
-static inline void mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {
-  if (mi_block_ptr_is_guarded(block, p)) { mi_block_unguard(page, block, p); }
+static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {
+  if (mi_block_ptr_is_guarded(block, p)) { 
+    mi_block_unguard(page, block, p); 
+    return true;
+  }
+  else {
+    return false;
+  }
 }
 #else
-static inline void mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {
+static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) {
   MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p);
+  return false;
 }
 #endif
 
@@ -124,16 +135,16 @@ static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, voi
 static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_attr_noexcept {
   mi_assert_internal(p!=NULL && page != NULL);
   mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));
-  mi_block_check_unguard(page, block, p);
-  mi_free_block_local(page, block, true /* track stats */, true /* check for a full page */);
+  const bool was_guarded = mi_block_check_unguard(page, block, p);
+  mi_free_block_local(page, block, was_guarded, true /* track stats */, true /* check for a full page */);
 }
 
 // free a pointer owned by another thread (page parameter comes first for better codegen)
 static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p) mi_attr_noexcept {
   mi_assert_internal(p!=NULL && page != NULL);
   mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p));
-  mi_block_check_unguard(page, block, p);
-  mi_free_block_mt(page, block);
+  const bool was_guarded = mi_block_check_unguard(page, block, p);
+  mi_free_block_mt(page, block, was_guarded);
 }
 
 // generic free (for runtime integration)
@@ -165,9 +176,8 @@ static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg)
 
 // Free a block
 // Fast path written carefully to prevent register spilling on the stack
-static mi_decl_forceinline void mi_free_ex(void* p, size_t* usable)  
+static mi_decl_forceinline void mi_free_ex(void* p, size_t* usable, mi_page_t* page)  
 {
-  mi_page_t* const page = mi_validate_ptr_page(p,"mi_free");
   if mi_unlikely(page==NULL) return;  // page will be NULL if p==NULL
   mi_assert_internal(p!=NULL && page!=NULL);
   if (usable!=NULL) { *usable = mi_page_usable_block_size(page); }
@@ -176,7 +186,7 @@ static mi_decl_forceinline void mi_free_ex(void* p, size_t* usable)
   if mi_likely(xtid == 0) {                        // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0`
     // thread-local, aligned, and not a full page
     mi_block_t* const block = mi_validate_block_from_ptr(page,p);
-    mi_free_block_local(page, block, true /* track stats */, false /* no need to check if the page is full */);
+    mi_free_block_local(page, block, false /* was guarded */, true /* track stats */, false /* no need to check if the page is full */);
   }
   else if (xtid <= MI_PAGE_FLAG_MASK) {            // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0`
     // page is local, but is full or contains (inner) aligned blocks; use generic path
@@ -186,7 +196,7 @@ static mi_decl_forceinline void mi_free_ex(void* p, size_t* usable)
   else if ((xtid & MI_PAGE_FLAG_MASK) == 0) {      // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0`
     // blocks are aligned (and not a full page); push on the thread_free list
     mi_block_t* const block = mi_validate_block_from_ptr(page,p);
-    mi_free_block_mt(page,block);
+    mi_free_block_mt(page,block,false /* was_guarded */);
   }
   else {
     // page is full or contains (inner) aligned blocks; use generic multi-thread path
@@ -195,13 +205,39 @@ static mi_decl_forceinline void mi_free_ex(void* p, size_t* usable)
 }
 
 void mi_free(void* p) mi_attr_noexcept {
-  mi_free_ex(p, NULL);
+  mi_page_t* const page = mi_validate_ptr_page(p,"mi_free");  
+  mi_free_ex(p, NULL, page);
 }
 
 void mi_ufree(void* p, size_t* usable) mi_attr_noexcept {
-  mi_free_ex(p, usable);
+  mi_page_t* const page = mi_validate_ptr_page(p,"mi_ufree");  
+  mi_free_ex(p, usable, page);
+}
+
+void mi_free_small(void* p) mi_attr_noexcept {
+  // We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`.
+  // If we keep page info in front of the page area for small objects, we can find the info
+  // just by aligning down the pointer instead of looking it up in the page map.
+  #if MI_PAGE_META_ALIGNED_FREE_SMALL 
+    #if MI_GUARDED 
+    #warning "MI_PAGE_META_ALIGNED_FREE_SMALL ignored as MI_GUARDED is defined"
+    mi_free(p);
+    #elif MI_ARENA_SLICE_ALIGN < MI_SMALL_PAGE_SIZE
+    #warning "MI_PAGE_META_ALIGNED_FREE_SMALL ignored as the MI_ARENA_SLICE_ALIGN is less than the small page size"
+    mi_free(p);
+    #else
+      mi_page_t* const page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE);
+      mi_assert(page == mi_validate_ptr_page(p,"mi_free_small"));
+      mi_assert((void*)page == _mi_align_down_ptr(page->page_start,MI_SMALL_PAGE_SIZE));
+      mi_assert(page->block_size <= MI_SMALL_SIZE_MAX);  // note: not `MI_SMALL_MAX_OBJ_SIZE` as we need to match `mi_(heap_)malloc_small`
+      mi_free_ex(p, NULL, page);
+    #endif
+  #else
+  mi_free(p);
+  #endif  
 }
 
+
 // --------------------------------------------------------------------------------------------
 // `mi_free_try_collect_mt`: Potentially collect a page in a free in an abandoned page.
 // 1. if the page becomes empty, free it
@@ -231,7 +267,7 @@ static bool mi_abandoned_page_try_reabandon_to_mapped(mi_page_t* page)
   return _mi_arenas_page_try_reabandon_to_mapped(page);
 }
 
-// Release ownership of a page. This may free or reabandond the page if other blocks are concurrently
+// Release ownership of a page. This may free or reabandoned the page if other blocks are concurrently
 // freed in the meantime. Returns `true` if the page was freed.
 // By passing the captured `expected_thread_free`, we can often avoid calling `mi_page_free_collect`.
 static void mi_abandoned_page_unown_from_free(mi_page_t* page, mi_block_t* expected_thread_free) {
@@ -277,7 +313,7 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long
   mi_assert_internal(mi_page_is_owned(page));
   mi_assert_internal(mi_page_is_abandoned(page));
   mi_assert_internal(!mi_page_all_free(page));
-  mi_assert_internal(page->block_size <= MI_SMALL_SIZE_MAX);
+  mi_assert_internal(page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE);
   mi_assert_internal(reclaim_on_free >= 0);
 
   // dont reclaim if we just have terminated this thread and we should
@@ -298,7 +334,7 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long
   else if (reclaim_on_free == 1 &&               // if cross-thread is allowed
             !theap->tld->is_in_threadpool &&      // and we are not part of a threadpool
             !mi_page_is_mostly_used(page) &&     // and the page is not too full
-            _mi_arena_memid_is_suitable(page->memid, theap->heap->exclusive_arena)) {   // and it fits our memory
+            _mi_arena_memid_is_suitable(page->memid, _mi_theap_heap(theap)->exclusive_arena)) {   // and it fits our memory
     // across threads
     max_reclaim = _mi_option_get_fast(mi_option_page_cross_thread_max_reclaim);
   }
@@ -344,7 +380,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t*
 
   // try to: 1. free it, 2. reclaim it, or 3. reabandon it to be mapped
   if (mi_abandoned_page_try_free(page)) return;
-  if (page->block_size <= MI_SMALL_SIZE_MAX && reclaim_on_free >= 0) {  // early test for better codegen
+  if (page->block_size <= MI_MEDIUM_MAX_OBJ_SIZE && reclaim_on_free >= 0) {  // early test for better codegen
     if (mi_abandoned_page_try_reclaim(page, reclaim_on_free)) return;
   }
   if (mi_abandoned_page_try_reabandon_to_mapped(page)) return;
@@ -361,15 +397,11 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t*
 // Bytes available in a block
 static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p) mi_attr_noexcept {
   const mi_block_t* block = _mi_page_ptr_unalign(page, p);
-  const size_t size = mi_page_usable_size_of(page, block);
+  const bool is_guarded = mi_block_ptr_is_guarded(block,p);
+  const size_t size = mi_page_usable_size_of(page, block, is_guarded);
   const ptrdiff_t adjust = (uint8_t*)p - (uint8_t*)block;
   mi_assert_internal(adjust >= 0 && (size_t)adjust <= size);
-  const size_t aligned_size = (size - adjust);
-  #if MI_GUARDED
-  if (mi_block_ptr_is_guarded(block, p)) {
-    return aligned_size - _mi_os_page_size();
-  }
-  #endif
+  const size_t aligned_size = (size - adjust);  
   return aligned_size;
 }
 
@@ -377,7 +409,7 @@ static inline size_t _mi_usable_size(const void* p, const mi_page_t* page) mi_at
   if mi_unlikely(page==NULL) return 0;
   if mi_likely(!mi_page_has_interior_pointers(page)) {
     const mi_block_t* block = (const mi_block_t*)p;
-    return mi_page_usable_size_of(page, block);
+    return mi_page_usable_size_of(page, block, false /* is guarded */);
   }
   else {
     // split out to separate routine for improved code generation
@@ -489,12 +521,18 @@ static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* bloc
 }
 
 // Return the exact usable size of a block.
-static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block) {
-  size_t bsize;
-  size_t delta;
-  bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
-  mi_assert_internal(ok); mi_assert_internal(delta <= bsize);
-  return (ok ? bsize - delta : 0);
+static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {
+  if (is_guarded) {
+    const size_t bsize = mi_page_block_size(page);
+    return (bsize - _mi_os_page_size());
+  }
+  else {
+    size_t bsize;
+    size_t delta;
+    bool ok = mi_page_decode_padding(page, block, &delta, &bsize);
+    mi_assert_internal(ok); mi_assert_internal(delta <= bsize);
+    return (ok ? bsize - delta : 0);
+  }
 }
 
 // When a non-thread-local block is freed, it becomes part of the thread delayed free
@@ -517,15 +555,13 @@ void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const si
   mi_track_mem_noaccess(padding,sizeof(mi_padding_t));
 }
 #else
-static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block) {
-  MI_UNUSED(block);
+static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool is_guarded) {
+  MI_UNUSED(is_guarded); MI_UNUSED(block);
   return mi_page_usable_block_size(page);
 }
 
 void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) {
-  MI_UNUSED(page);
-  MI_UNUSED(block);
-  MI_UNUSED(min_size);
+  MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(min_size);
 }
 #endif