@shd101wyy/yo 0.1.24 → 0.1.26

package/vendor/mimalloc/include/mimalloc/atomic.h

@@ -97,7 +97,6 @@ terms of the MIT license. A copy of the license can be found in the file
 #define mi_atomic_increment_acq_rel(p)           mi_atomic_add_acq_rel(p,(uintptr_t)1)
 #define mi_atomic_decrement_acq_rel(p)           mi_atomic_sub_acq_rel(p,(uintptr_t)1)
 
-static inline void mi_atomic_yield(void);
 static inline intptr_t mi_atomic_addi(_Atomic(intptr_t)*p, intptr_t add);
 static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub);
 
@@ -159,14 +158,18 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) {
 
 #elif defined(_MSC_VER)
 
-// Legacy MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics.
+// Deprecated: MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics.
+// It is recommended to always compile as C++ when using MSVC.
+
 #include <intrin.h>
 #ifdef _WIN64
-typedef LONG64   msc_intptr_t;
-#define MI_64(f) f##64
+typedef LONG64        msc_intptr_t;
+#define MI_MSC_64(f)  f##64
+#define MI_MSC_XX(f)  f##64
 #else
-typedef LONG     msc_intptr_t;
-#define MI_64(f) f
+typedef LONG          msc_intptr_t;
+#define MI_MSC_64(f)  f
+#define MI_MSC_XX(f)  f##32
 #endif
 
 typedef enum mi_memory_order_e {
@@ -180,23 +183,23 @@ typedef enum mi_memory_order_e {
 
 static inline uintptr_t mi_atomic_fetch_add_explicit(_Atomic(uintptr_t)*p, uintptr_t add, mi_memory_order mo) {
   (void)(mo);
-  return (uintptr_t)MI_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add);
+  return (uintptr_t)MI_MSC_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, (msc_intptr_t)add);
 }
 static inline uintptr_t mi_atomic_fetch_sub_explicit(_Atomic(uintptr_t)*p, uintptr_t sub, mi_memory_order mo) {
   (void)(mo);
-  return (uintptr_t)MI_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, -((msc_intptr_t)sub));
+  return (uintptr_t)MI_MSC_64(_InterlockedExchangeAdd)((volatile msc_intptr_t*)p, -((msc_intptr_t)sub));
 }
 static inline uintptr_t mi_atomic_fetch_and_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
   (void)(mo);
-  return (uintptr_t)MI_64(_InterlockedAnd)((volatile msc_intptr_t*)p, (msc_intptr_t)x);
+  return (uintptr_t)MI_MSC_64(_InterlockedAnd)((volatile msc_intptr_t*)p, (msc_intptr_t)x);
 }
 static inline uintptr_t mi_atomic_fetch_or_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
   (void)(mo);
-  return (uintptr_t)MI_64(_InterlockedOr)((volatile msc_intptr_t*)p, (msc_intptr_t)x);
+  return (uintptr_t)MI_MSC_64(_InterlockedOr)((volatile msc_intptr_t*)p, (msc_intptr_t)x);
 }
 static inline bool mi_atomic_compare_exchange_strong_explicit(_Atomic(uintptr_t)*p, uintptr_t* expected, uintptr_t desired, mi_memory_order mo1, mi_memory_order mo2) {
   (void)(mo1); (void)(mo2);
-  uintptr_t read = (uintptr_t)MI_64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)(*expected));
+  const uintptr_t read = (uintptr_t)MI_MSC_64(_InterlockedCompareExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)desired, (msc_intptr_t)(*expected));
   if (read == *expected) {
     return true;
   }
@@ -210,68 +213,119 @@ static inline bool mi_atomic_compare_exchange_weak_explicit(_Atomic(uintptr_t)*p
 }
 static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintptr_t exchange, mi_memory_order mo) {
   (void)(mo);
-  return (uintptr_t)MI_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange);
+  return (uintptr_t)MI_MSC_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange);
 }
 static inline void mi_atomic_thread_fence(mi_memory_order mo) {
   (void)(mo);
   _Atomic(uintptr_t) x = 0;
   mi_atomic_exchange_explicit(&x, 1, mo);
 }
+
 static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) {
   (void)(mo);
-#if defined(_M_IX86) || defined(_M_X64)
-  return *p;
-#else
-  uintptr_t x = *p;
-  if (mo > mi_memory_order_relaxed) {
-    while (!mi_atomic_compare_exchange_weak_explicit((_Atomic(uintptr_t)*)p, &x, x, mo, mi_memory_order_relaxed)) { /* nothing */ };
-  }
-  return x;
-#endif
+  // assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc
+  #if defined(_M_IX86) || defined(_M_X64)
+    return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
+  #elif defined(_M_ARM) || defined(_M_ARM64)
+    if (mo == mi_memory_order_relaxed) {
+      return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
+    }
+    else if (mo <= mi_memory_order_acquire) {
+      return MI_MSC_XX(__ldar)((volatile const uintptr_t*)p);
+    }
+    else {
+      const uintptr_t u = (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
+      __dmb(15);  // _ARM(64)_BARRIER_SY
+      return u;
+    }
+  #else
+    #warning "define mi_atomic_load_explicit for MSVC C compilation on this platform (which should be readonly, see issue #1277)"
+    return MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p);
+  #endif
 }
 static inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, mi_memory_order mo) {
   (void)(mo);
-#if defined(_M_IX86) || defined(_M_X64)
-  *p = x;
-#else
-  mi_atomic_exchange_explicit(p, x, mo);
-#endif
+  // assert(mo<=mi_memory_order_release); // others are not used by mimalloc
+  #if defined(_M_IX86) || defined(_M_X64)
+    MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x);
+  #elif defined(_M_ARM) || defined(_M_ARM64)
+    if (mo == mi_memory_order_relaxed) {
+      MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x);
+    }
+    else if (mo <= mi_memory_order_release) {
+      MI_MSC_XX(__stlr)((volatile uintptr_t*)p,x);
+    }
+    else {
+      mi_atomic_exchange_explicit(p, x, mo);
+    }
+  #else
+    mi_atomic_exchange_explicit(p, x, mo);
+  #endif
 }
+
 static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_order mo) {
   (void)(mo);
-#if defined(_M_X64)
-  return *p;
-#else
-  int64_t old = *p;
-  int64_t x = old;
-  while ((old = InterlockedCompareExchange64(p, x, old)) != x) {
-    x = old;
-  }
-  return x;
-#endif
+  // assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc
+  #if defined(_M_IX86) || defined(_M_X64)
+    return __iso_volatile_load64((volatile const int64_t*)p);
+  #elif defined(_M_ARM) || defined(_M_ARM64)
+    if (mo == mi_memory_order_relaxed) {
+      return __iso_volatile_load64((volatile const int64_t*)p);
+    }
+    #if defined(_M_ARM64)
+    else if (mo <= mi_memory_order_acquire) {
+      return __ldar64((volatile const uintptr_t*)p);
+    }
+    #endif
+    else {
+      const int64_t i = __iso_volatile_load64((volatile const int64_t*)p);
+      __dmb(15);  // _ARM(64)_BARRIER_SY
+      return i;
+    }
+  #else
+    #warning "define mi_atomic_loadi64_explicit for MSVC C compilation on this platform (which should be readonly, see issue #1277)"
+    return __iso_volatile_load64((volatile const int64_t*)p);
+  #endif
 }
+
 static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi_memory_order mo) {
   (void)(mo);
-#if defined(x_M_IX86) || defined(_M_X64)
-  *p = x;
-#else
-  InterlockedExchange64(p, x);
-#endif
+  // assert(mo<=mi_memory_order_release); // others are not used by mimalloc
+  #if defined(_M_IX86) || defined(_M_X64)
+    __iso_volatile_store64((volatile int64_t*)p,x);
+  #elif defined(_M_ARM) || defined(_M_ARM64)
+    if (mo == mi_memory_order_relaxed) {
+      __iso_volatile_store64((volatile int64_t*)p,x);
+    }
+    #if defined(_M_ARM64)
+    else if (mo == mi_memory_order_release) {
+      __stlr64((volatile uint64_t*)p, (uint64_t)x);
+    }
+    #endif
+    else {
+      InterlockedExchange64(p, x);
+    }
+  #else
+    InterlockedExchange64(p, x);
+  #endif
 }
 
 // These are used by the statistics
 static inline int64_t mi_atomic_addi64_relaxed(volatile _Atomic(int64_t)*p, int64_t add) {
-#ifdef _WIN64
-  return (int64_t)mi_atomic_addi((int64_t*)p, add);
-#else
-  int64_t current;
-  int64_t sum;
-  do {
-    current = *p;
-    sum = current + add;
-  } while (_InterlockedCompareExchange64(p, sum, current) != current);
-  return current;
-#endif
+  #ifdef _WIN64
+    return (int64_t)mi_atomic_addi((int64_t*)p, add);
+  #elif defined(_M_ARM)
+    return _InterlockedExchangeAdd64(p, add);
+  #else
+    // x86
+    int64_t current;
+    int64_t sum;
+    do {
+      current = __iso_volatile_load64((volatile const int64_t*)p);
+      sum = current + add;
+    } while (_InterlockedCompareExchange64(p, sum, current) != current);
+    return current;
+  #endif
 }
 
 static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const volatile int64_t* padd) {
@@ -293,7 +347,7 @@ static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t*)p, int64_t
 }
 
 static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, int64_t* exp, int64_t des) {
-  int64_t read = _InterlockedCompareExchange64(p, des, *exp);
+  const int64_t read = _InterlockedCompareExchange64(p, des, *exp);
   if (read == *exp) {
     return true;
   }
@@ -337,21 +391,12 @@ static inline intptr_t mi_atomic_subi(_Atomic(intptr_t)*p, intptr_t sub) {
 
 
 // ----------------------------------------------------------------------
-// Once and Guard
+// Guard
 // ----------------------------------------------------------------------
 
-typedef _Atomic(uintptr_t) mi_atomic_once_t;
-
-// Returns true only on the first invocation
-static inline bool mi_atomic_once( mi_atomic_once_t* once ) {
-  if (mi_atomic_load_relaxed(once) != 0) return false;     // quick test
-  uintptr_t expected = 0;
-  return mi_atomic_cas_strong_acq_rel(once, &expected, (uintptr_t)1); // try to set to 1
-}
-
 typedef _Atomic(uintptr_t) mi_atomic_guard_t;
 
-// Allows only one thread to execute at a time
+// Allows only one thread to execute at a time (without blocking anyone)
 #define mi_atomic_guard(guard) \
   uintptr_t _mi_guard_expected = 0; \
   for(bool _mi_guard_once = true; \
@@ -359,77 +404,6 @@ typedef _Atomic(uintptr_t) mi_atomic_guard_t;
       (mi_atomic_store_release(guard,(uintptr_t)0), _mi_guard_once = false) )
 
 
-
-// ----------------------------------------------------------------------
-// Yield
-// ----------------------------------------------------------------------
-
-#if defined(__cplusplus)
-#include <thread>
-static inline void mi_atomic_yield(void) {
-  std::this_thread::yield();
-}
-#elif defined(_WIN32)
-static inline void mi_atomic_yield(void) {
-  YieldProcessor();
-}
-#elif defined(__SSE2__)
-#include <emmintrin.h>
-static inline void mi_atomic_yield(void) {
-  _mm_pause();
-}
-#elif (defined(__GNUC__) || defined(__clang__)) && \
-      (defined(__x86_64__) || defined(__i386__) || \
-       defined(__aarch64__) || defined(__arm__) || \
-       defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) || defined(__POWERPC__))
-#if defined(__x86_64__) || defined(__i386__)
-static inline void mi_atomic_yield(void) {
-  __asm__ volatile ("pause" ::: "memory");
-}
-#elif defined(__aarch64__)
-static inline void mi_atomic_yield(void) {
-  __asm__ volatile("wfe");
-}
-#elif defined(__arm__)
-#if __ARM_ARCH >= 7
-static inline void mi_atomic_yield(void) {
-  __asm__ volatile("yield" ::: "memory");
-}
-#else
-static inline void mi_atomic_yield(void) {
-  __asm__ volatile ("nop" ::: "memory");
-}
-#endif
-#elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) || defined(__POWERPC__)
-#ifdef __APPLE__
-static inline void mi_atomic_yield(void) {
-  __asm__ volatile ("or r27,r27,r27" ::: "memory");
-}
-#else
-static inline void mi_atomic_yield(void) {
-  __asm__ __volatile__ ("or 27,27,27" ::: "memory");
-}
-#endif
-#endif
-#elif defined(__sun)
-// Fallback for other archs
-#include <synch.h>
-static inline void mi_atomic_yield(void) {
-  smt_pause();
-}
-#elif defined(__wasi__)
-#include <sched.h>
-static inline void mi_atomic_yield(void) {
-  sched_yield();
-}
-#else
-#include <unistd.h>
-static inline void mi_atomic_yield(void) {
-  sleep(0);
-}
-#endif
-
-
 // ----------------------------------------------------------------------
 // Locks
 // These should be light-weight in-process only locks.
@@ -439,95 +413,90 @@ static inline void mi_atomic_yield(void) {
 #pragma warning(disable:26110)  // unlock with holding lock
 #endif
 
-#define mi_lock(lock)    for(bool _go = (mi_lock_acquire(lock),true); _go; (mi_lock_release(lock), _go=false) )
+#define mi_lock(lock)                  for(bool _mi_go = (mi_lock_acquire(lock),true); _mi_go; (mi_lock_release(lock), _mi_go=false) )
+#define mi_lock_maybe(lock,acquire)    for(bool _mi_go = (acquire ? (mi_lock_acquire(lock),true) : true); _mi_go; _mi_go = (acquire ? (mi_lock_release(lock),false) : false) )
+
 
 #if defined(_WIN32)
 
-#if 1
-#define mi_lock_t  SRWLOCK   // slim reader-writer lock
+typedef struct mi_lock_s {
+  SRWLOCK mutex;    // slim reader-writer lock
+} mi_lock_t;
 
-static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
-  return TryAcquireSRWLockExclusive(lock);
-}
-static inline void mi_lock_acquire(mi_lock_t* lock) {
-  AcquireSRWLockExclusive(lock);
-}
-static inline void mi_lock_release(mi_lock_t* lock) {
-  ReleaseSRWLockExclusive(lock);
-}
-static inline void mi_lock_init(mi_lock_t* lock) {
-  InitializeSRWLock(lock);
-}
-static inline void mi_lock_done(mi_lock_t* lock) {
-  (void)(lock);
-}
-
-#else
-#define mi_lock_t  CRITICAL_SECTION
+#define MI_LOCK_INITIALIZER   { SRWLOCK_INIT }
 
 static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
-  return TryEnterCriticalSection(lock);
+  return TryAcquireSRWLockExclusive(&lock->mutex);
 }
 static inline void mi_lock_acquire(mi_lock_t* lock) {
-  EnterCriticalSection(lock);
+  AcquireSRWLockExclusive(&lock->mutex);
 }
 static inline void mi_lock_release(mi_lock_t* lock) {
-  LeaveCriticalSection(lock);
+  ReleaseSRWLockExclusive(&lock->mutex);
 }
 static inline void mi_lock_init(mi_lock_t* lock) {
-  InitializeCriticalSection(lock);
+  InitializeSRWLock(&lock->mutex);
 }
 static inline void mi_lock_done(mi_lock_t* lock) {
-  DeleteCriticalSection(lock);
+  (void)(lock);
 }
 
-#endif
-
 #elif defined(MI_USE_PTHREADS)
 
 #include <string.h> // memcpy
 void _mi_error_message(int err, const char* fmt, ...);
 
-#define mi_lock_t  pthread_mutex_t
+typedef struct mi_lock_s {
+  pthread_mutex_t mutex;
+} mi_lock_t;
+
+#define MI_LOCK_INITIALIZER { PTHREAD_MUTEX_INITIALIZER }
 
 static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
-  return (pthread_mutex_trylock(lock) == 0);
+  return (pthread_mutex_trylock(&lock->mutex) == 0);
 }
 static inline void mi_lock_acquire(mi_lock_t* lock) {
-  const int err = pthread_mutex_lock(lock);
+  const int err = pthread_mutex_lock(&lock->mutex);
   if (err != 0) {
     _mi_error_message(err, "internal error: lock cannot be acquired (err %i)\n", err);
   }
 }
 static inline void mi_lock_release(mi_lock_t* lock) {
-  pthread_mutex_unlock(lock);
+  pthread_mutex_unlock(&lock->mutex);
 }
 static inline void mi_lock_init(mi_lock_t* lock) {
   if(lock==NULL) return;
-  // use instead of pthread_mutex_init since that can cause allocation on some platforms (and recursively initialize)
-  const mi_lock_t temp_lock = PTHREAD_MUTEX_INITIALIZER;  
-  memcpy(lock,&temp_lock,sizeof(temp_lock));
+  // use this instead of pthread_mutex_init since that can cause allocation on some platforms (and recursively initialize)
+  const pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+  memcpy(&lock->mutex,&mutex,sizeof(mutex));
 }
 static inline void mi_lock_done(mi_lock_t* lock) {
-  pthread_mutex_destroy(lock);
+  pthread_mutex_destroy(&lock->mutex);
 }
 
 #elif defined(__cplusplus)
 
+#include <thread>
 #include <mutex>
-#define mi_lock_t  std::mutex
+#include <new>
+
+typedef struct mi_lock_s {
+  std::mutex mutex;
+} mi_lock_t;
+
+#define MI_LOCK_INITIALIZER   { }
 
 static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
-  return lock->try_lock();
+  return lock->mutex.try_lock();
 }
 static inline void mi_lock_acquire(mi_lock_t* lock) {
-  lock->lock();
+  lock->mutex.lock();
 }
 static inline void mi_lock_release(mi_lock_t* lock) {
-  lock->unlock();
+  lock->mutex.unlock();
 }
 static inline void mi_lock_init(mi_lock_t* lock) {
-  (void)(lock);
+  new(&lock->mutex) std::mutex();
 }
 static inline void mi_lock_done(mi_lock_t* lock) {
   (void)(lock);
@@ -537,21 +506,32 @@ static inline void mi_lock_done(mi_lock_t* lock) {
 
 // fall back to poor man's locks.
 // this should only be the case in a single-threaded environment (like __wasi__)
+#include <errno.h>
+#ifndef EFAULT
+#define EFAULT (14)
+#endif
+void _mi_error_message(int err, const char* fmt, ...);
+void _mi_prim_thread_yield(void);
+
+typedef struct mi_lock_s {
+  _Atomic(uintptr_t) mutex;
+} mi_lock_t;
 
-#define mi_lock_t  _Atomic(uintptr_t)
+#define MI_LOCK_INITIALIZER  { MI_ATOMIC_VAR_INIT(0) }
 
 static inline bool mi_lock_try_acquire(mi_lock_t* lock) {
   uintptr_t expected = 0;
-  return mi_atomic_cas_strong_acq_rel(lock, &expected, (uintptr_t)1);
+  return mi_atomic_cas_strong_acq_rel(&lock->mutex, &expected, (uintptr_t)1);
 }
 static inline void mi_lock_acquire(mi_lock_t* lock) {
-  for (int i = 0; i < 1000; i++) {  // for at most 1000 tries?
+  for (int i = 0; i < 10000; i++) {  // for at most 10000 tries?
     if (mi_lock_try_acquire(lock)) return;
-    mi_atomic_yield();
+    _mi_prim_thread_yield();
   }
+  _mi_error_message(EFAULT, "internal error: lock cannot be acquired (due to lack of native lock primitives)\n");
 }
 static inline void mi_lock_release(mi_lock_t* lock) {
-  mi_atomic_store_release(lock, (uintptr_t)0);
+  mi_atomic_store_release(&lock->mutex, (uintptr_t)0);
 }
 static inline void mi_lock_init(mi_lock_t* lock) {
   mi_lock_release(lock);
@@ -563,4 +543,20 @@ static inline void mi_lock_done(mi_lock_t* lock) {
 #endif
 
 
-#endif // MI_ATOMIC_H
+typedef struct mi_atomic_once_s {
+  _Atomic(uintptr_t) tid;
+  mi_lock_t          lock;
+} mi_atomic_once_t;
+
+// Returns `true` only on the first invocation, signifying we can execute an action once.
+// If it returns `true`, the caller should call `_mi_atomic_once_release` after performing the action.
+// Other threads (than the initial thread that entered) will block until `_mi_atomic_once_release` has been called.
+bool _mi_atomic_once_enter(mi_atomic_once_t* once);        // defined in `libc.c`
+void _mi_atomic_once_release(mi_atomic_once_t* once);      // defined in `libc.c`
+
+#define mi_atomic_do_once  \
+  static mi_atomic_once_t _mi_once = { MI_ATOMIC_VAR_INIT(0), MI_LOCK_INITIALIZER }; \
+  for(bool _mi_exec = _mi_atomic_once_enter(&_mi_once); _mi_exec; (_mi_atomic_once_release(&_mi_once),_mi_exec=false))
+
+
+#endif // __MIMALLOC_ATOMIC_H

package/vendor/mimalloc/include/mimalloc/bits.h

@@ -108,6 +108,9 @@ typedef int32_t  mi_ssize_t;
 #if MI_ARCH_X64 && (defined(__AVX2__) || defined(__BMI2__)) && !defined(__BMI1__) // msvc
 #define __BMI1__  1
 #endif
+#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__LZCNT__) // msvc
+#define __LZCNT__  1
+#endif
 
 // Define big endian if needed
 // #define MI_BIG_ENDIAN  1
@@ -123,9 +126,9 @@ typedef int32_t  mi_ssize_t;
 #define MI_MAX_VABITS     (32)
 #endif
 
-// use a flat page-map (or a 2-level one)
+// use a flat page-map or a 2-level one
 #ifndef MI_PAGE_MAP_FLAT
-#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) 
+#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && MI_SECURE==0 && !MI_PAGE_META_IS_SEPARATED
 #define MI_PAGE_MAP_FLAT  1
 #else
 #define MI_PAGE_MAP_FLAT  0
@@ -222,11 +225,11 @@ static inline size_t mi_ctz(size_t x) {
 }
 
 static inline size_t mi_clz(size_t x) {
-  #if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__) // on x64 lzcnt is defined for 0
+  #if defined(__GNUC__) && MI_ARCH_X64 && defined(__LZCNT__) // on x64 lzcnt is defined for 0
     size_t r;
     __asm ("lzcnt\t%1, %0" : "=r"(r) : "r"(x) : "cc");
     return r;
-  #elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__BMI1__) 
+  #elif defined(_MSC_VER) && MI_ARCH_X64 && defined(__LZCNT__) 
     return _lzcnt_u64(x);
   #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
     unsigned long idx;
@@ -273,12 +276,7 @@ static inline bool mi_bsf(size_t x, size_t* idx) {
 // return false if `x==0` (with `*idx` undefined) and true otherwise,
 // with the `idx` is set to the bit index (`0 <= *idx < MI_BFIELD_BITS`).
 static inline bool mi_bsr(size_t x, size_t* idx) {
-  #if defined(__GNUC__) && MI_ARCH_X64 && defined(__BMI1__)  && (!defined(__clang_major__) || __clang_major__ >= 9)
-    // on x64 the carry flag is set on zero which gives better codegen
-    bool is_zero;
-    __asm ("lzcnt\t%2, %1" : "=@ccc"(is_zero), "=r"(*idx) : "r"(x) : "cc");
-    return !is_zero;
-  #elif 0 && defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
+  #if 0 && defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_ARM64 || MI_ARCH_ARM32)
     unsigned long i;
     return (mi_msc_builtinz(_BitScanReverse)(&i, x) ? (*idx = (size_t)i, true) : false);
   #else