@img/sharp-libvips-dev-wasm32 1.3.0-rc.1 → 1.3.0-rc.10

package/include/hwy/base.h

@@ -22,8 +22,8 @@
 #include <stddef.h>
 #include <stdint.h>
 #if defined(HWY_HEADER_ONLY)
-#include <cstdarg>
-#include <cstdio>
+#include <stdarg.h>
+#include <stdio.h>
 #endif
 
 #if !defined(HWY_NO_LIBCXX)
@@ -33,9 +33,10 @@
 #include "hwy/detect_compiler_arch.h"
 #include "hwy/highway_export.h"
 
-// API version (https://semver.org/); keep in sync with CMakeLists.txt.
+// API version (https://semver.org/); keep in sync with CMakeLists.txt and
+// meson.build.
 #define HWY_MAJOR 1
-#define HWY_MINOR 3
+#define HWY_MINOR 4
 #define HWY_PATCH 0
 
 // True if the Highway version >= major.minor.0. Added in 1.2.0.
@@ -57,7 +58,7 @@
 
 #endif  // !HWY_IDE
 
-#if (HWY_ARCH_X86 && !defined(HWY_NO_LIBCXX)) || HWY_COMPILER_MSVC
+#if !defined(HWY_NO_LIBCXX) || HWY_COMPILER_MSVC
 #include <atomic>
 #endif
 
@@ -148,6 +149,21 @@
 
 #endif  // !HWY_COMPILER_MSVC
 
+#if (HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 1200) || \
+    (HWY_COMPILER_ICC && !HWY_COMPILER_ICX)
+// The use of __attribute__((unused)) in private class member variables triggers
+// a compiler warning with GCC 11 and earlier and ICC
+
+// GCC 11 and earlier and ICC also do not emit -Wunused-private-field warnings
+// for unused private class member variables
+#define HWY_MEMBER_VAR_MAYBE_UNUSED
+#else
+// Clang and ICX need __attribute__((unused)) in unused private class member
+// variables to suppress -Wunused-private-field warnings unless this warning is
+// ignored by using HWY_DIAGNOSTICS_OFF
+#define HWY_MEMBER_VAR_MAYBE_UNUSED HWY_MAYBE_UNUSED
+#endif
+
 //------------------------------------------------------------------------------
 // Builtin/attributes (no more #include after this point due to namespace!)
 
@@ -202,7 +218,9 @@ namespace hwy {
 //------------------------------------------------------------------------------
 // Macros
 
-#define HWY_API static HWY_INLINE HWY_FLATTEN HWY_MAYBE_UNUSED
+// Note: it is safe to remove `static` for users who want to use modules, but
+// that might be a breaking change for some users, hence we do not by default.
+#define HWY_API static HWY_INLINE HWY_FLATTEN
 
 #define HWY_CONCAT_IMPL(a, b) a##b
 #define HWY_CONCAT(a, b) HWY_CONCAT_IMPL(a, b)
@@ -349,60 +367,20 @@ HWY_DLLEXPORT HWY_NORETURN void HWY_FORMAT(3, 4)
   } while (0)
 #define HWY_ASSERT(condition) HWY_ASSERT_M(condition, "")
 
-#if HWY_HAS_FEATURE(memory_sanitizer) || defined(MEMORY_SANITIZER) || \
-    defined(__SANITIZE_MEMORY__)
-#define HWY_IS_MSAN 1
-#else
-#define HWY_IS_MSAN 0
-#endif
-
-#if HWY_HAS_FEATURE(address_sanitizer) || defined(ADDRESS_SANITIZER) || \
-    defined(__SANITIZE_ADDRESS__)
-#define HWY_IS_ASAN 1
-#else
-#define HWY_IS_ASAN 0
-#endif
-
-#if HWY_HAS_FEATURE(hwaddress_sanitizer) || defined(HWADDRESS_SANITIZER) || \
-    defined(__SANITIZE_HWADDRESS__)
-#define HWY_IS_HWASAN 1
-#else
-#define HWY_IS_HWASAN 0
-#endif
-
-#if HWY_HAS_FEATURE(thread_sanitizer) || defined(THREAD_SANITIZER) || \
-    defined(__SANITIZE_THREAD__)
-#define HWY_IS_TSAN 1
-#else
-#define HWY_IS_TSAN 0
-#endif
-
-#if HWY_HAS_FEATURE(undefined_behavior_sanitizer) || \
-    defined(UNDEFINED_BEHAVIOR_SANITIZER)
-#define HWY_IS_UBSAN 1
-#else
-#define HWY_IS_UBSAN 0
-#endif
-
-// MSAN may cause lengthy build times or false positives e.g. in AVX3 DemoteTo.
-// You can disable MSAN by adding this attribute to the function that fails.
-#if HWY_IS_MSAN
-#define HWY_ATTR_NO_MSAN __attribute__((no_sanitize_memory))
-#else
-#define HWY_ATTR_NO_MSAN
-#endif
-
-#if HWY_IS_ASAN || HWY_IS_HWASAN || HWY_IS_MSAN || HWY_IS_TSAN || HWY_IS_UBSAN
-#define HWY_IS_SANITIZER 1
-#else
-#define HWY_IS_SANITIZER 0
-#endif
-
 // For enabling HWY_DASSERT and shortening tests in slower debug builds
+//
+// Note: `HWY_IS_UBSAN` is specifically excluded from engaging debug
+// builds. This is in service of Chromium's `-fsanitize=array-bounds` by
+// default, where we don't want Highway to unconditionally build in
+// debug mode.
+//
+// See also:
+// https://docs.google.com/document/d/1eCtY4AZF-SiFHxhIYWzEytdIx3C24de7ccD6Y5Gn2H8/edit?tab=t.9zkn85hr82ms#heading=h.efcshvfql42c
 #if !defined(HWY_IS_DEBUG_BUILD)
 // Clang does not define NDEBUG, but it and GCC define __OPTIMIZE__, and recent
 // MSVC defines NDEBUG (if not, could instead check _DEBUG).
-#if (!defined(__OPTIMIZE__) && !defined(NDEBUG)) || HWY_IS_SANITIZER || \
+#if (!defined(__OPTIMIZE__) && !defined(NDEBUG)) ||         \
+    (HWY_IS_ASAN || (HWY_IS_SANITIZER && !HWY_IS_UBSAN)) || \
     defined(__clang_analyzer__)
 #define HWY_IS_DEBUG_BUILD 1
 #else
@@ -455,21 +433,22 @@ HWY_API void CopySameSize(const From* HWY_RESTRICT from, To* HWY_RESTRICT to) {
   CopyBytes<sizeof(From)>(from, to);
 }
 
-template <size_t kBytes, typename To>
-HWY_API void ZeroBytes(To* to) {
+// Sets each byte to `byte_value`, which must be between 0 and 255.
+HWY_API void FillBytes(void* to, int byte_value, size_t num_bytes) {
 #if HWY_COMPILER_MSVC
-  memset(to, 0, kBytes);
+  memset(to, byte_value, num_bytes);
 #else
-  __builtin_memset(to, 0, kBytes);
+  __builtin_memset(to, byte_value, num_bytes);
 #endif
 }
 
 HWY_API void ZeroBytes(void* to, size_t num_bytes) {
-#if HWY_COMPILER_MSVC
-  memset(to, 0, num_bytes);
-#else
-  __builtin_memset(to, 0, num_bytes);
-#endif
+  FillBytes(to, 0, num_bytes);
+}
+
+template <size_t kBytes, typename To>
+HWY_API void ZeroBytes(To* to) {
+  ZeroBytes(to, kBytes);
 }
 
 //------------------------------------------------------------------------------
@@ -1699,28 +1678,14 @@ HWY_F16_CONSTEXPR inline std::partial_ordering operator<=>(
 //------------------------------------------------------------------------------
 // BF16 lane type
 
-// Compiler supports ACLE __bf16, not necessarily with operators.
-
-// Disable the __bf16 type on AArch64 with GCC 13 or earlier as there is a bug
-// in GCC 13 and earlier that sometimes causes BF16 constant values to be
-// incorrectly loaded on AArch64, and this GCC bug on AArch64 is
-// described at https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111867.
-
-#if HWY_ARCH_ARM_A64 && \
-    (HWY_COMPILER_CLANG >= 1700 || HWY_COMPILER_GCC_ACTUAL >= 1400)
-#define HWY_ARM_HAVE_SCALAR_BF16_TYPE 1
-#else
-#define HWY_ARM_HAVE_SCALAR_BF16_TYPE 0
-#endif
-
 // x86 compiler supports __bf16, not necessarily with operators.
 // Disable in debug builds due to clang miscompiles as of 2025-07-22: casting
 // bf16 <-> f32 in convert_test results in 0x2525 for 1.0 instead of 0x3f80.
 // Reported at https://github.com/llvm/llvm-project/issues/151692.
 #ifndef HWY_SSE2_HAVE_SCALAR_BF16_TYPE
-#if HWY_ARCH_X86 && defined(__SSE2__) &&                     \
-    ((HWY_COMPILER_CLANG >= 1700 && !HWY_COMPILER_CLANGCL && \
-      !HWY_IS_DEBUG_BUILD) ||                                \
+#if HWY_ARCH_X86 && defined(__SSE2__) &&                         \
+    ((HWY_COMPILER_CLANG >= 1700 && !HWY_COMPILER_CLANGCL &&     \
+      (!HWY_IS_DEBUG_BUILD || HWY_COMPILER3_CLANG >= 220101)) || \
      HWY_COMPILER_GCC_ACTUAL >= 1300)
 #define HWY_SSE2_HAVE_SCALAR_BF16_TYPE 1
 #else
@@ -1738,7 +1703,11 @@ HWY_F16_CONSTEXPR inline std::partial_ordering operator<=>(
 #ifndef HWY_HAVE_SCALAR_BF16_OPERATORS
 // Recent enough compiler also has operators. aarch64 clang 18 hits internal
 // compiler errors on bf16 ToString, hence only enable on GCC for now.
-#if HWY_HAVE_SCALAR_BF16_TYPE && (HWY_COMPILER_GCC_ACTUAL >= 1300)
+// GCC >= 13 will insert a function call to the __extendbfsf2 helper function
+// for scalar conversions from __bf16 to float. This is prohibitively expensive,
+// so refrain from using scalar BF16 operators on these compiler versions.
+// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=121853
+#if HWY_HAVE_SCALAR_BF16_TYPE && (HWY_COMPILER_GCC_ACTUAL >= 1700)
 #define HWY_HAVE_SCALAR_BF16_OPERATORS 1
 #else
 #define HWY_HAVE_SCALAR_BF16_OPERATORS 0
@@ -2734,7 +2703,7 @@ HWY_API constexpr TTo ConvertScalarTo(TFrom in) {
 template <typename T1, typename T2>
 constexpr inline T1 DivCeil(T1 a, T2 b) {
 #if HWY_CXX_LANG >= 201703L
-  HWY_DASSERT(b != 0);
+  HWY_DASSERT(b != T2{0});
 #endif
   return (a + b - 1) / b;
 }
@@ -2957,9 +2926,10 @@ HWY_INLINE constexpr T AddWithWraparound(T t, T2 n) {
 // 64 x 64 = 128 bit multiplication
 HWY_API uint64_t Mul128(uint64_t a, uint64_t b, uint64_t* HWY_RESTRICT upper) {
 #if defined(__SIZEOF_INT128__)
-  __uint128_t product = (__uint128_t)a * (__uint128_t)b;
-  *upper = (uint64_t)(product >> 64);
-  return (uint64_t)(product & 0xFFFFFFFFFFFFFFFFULL);
+  __uint128_t product =
+      static_cast<__uint128_t>(a) * static_cast<__uint128_t>(b);
+  *upper = static_cast<uint64_t>(product >> 64);
+  return static_cast<uint64_t>(product & 0xFFFFFFFFFFFFFFFFULL);
 #elif HWY_COMPILER_MSVC && HWY_ARCH_X86_64
   return _umul128(a, b, upper);
 #else
@@ -2976,9 +2946,9 @@ HWY_API uint64_t Mul128(uint64_t a, uint64_t b, uint64_t* HWY_RESTRICT upper) {
 
 HWY_API int64_t Mul128(int64_t a, int64_t b, int64_t* HWY_RESTRICT upper) {
 #if defined(__SIZEOF_INT128__)
-  __int128_t product = (__int128_t)a * (__int128_t)b;
-  *upper = (int64_t)(product >> 64);
-  return (int64_t)(product & 0xFFFFFFFFFFFFFFFFULL);
+  __int128_t product = static_cast<__int128_t>(a) * static_cast<__int128_t>(b);
+  *upper = static_cast<int64_t>(product >> 64);
+  return static_cast<int64_t>(product & 0xFFFFFFFFFFFFFFFFULL);
 #elif HWY_COMPILER_MSVC && HWY_ARCH_X86_64
   return _mul128(a, b, upper);
 #else

package/include/hwy/bit_set.h

@@ -0,0 +1,410 @@
+// Copyright 2024 Google LLC
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HIGHWAY_HWY_BIT_SET_H_
+#define HIGHWAY_HWY_BIT_SET_H_
+
+// Various BitSet for 64, up to 4096, or any number of bits.
+
+#include <stddef.h>
+
+#include <atomic>
+
+#include "hwy/base.h"
+
+namespace hwy {
+
+// 64-bit specialization of `std::bitset`, which lacks `Foreach`.
+class BitSet64 {
+ public:
+  constexpr size_t MaxSize() const { return 64; }
+
+  // No harm if `i` is already set.
+  void Set(size_t i) {
+    HWY_DASSERT(i < 64);
+    bits_ |= (1ULL << i);
+    HWY_DASSERT(Get(i));
+  }
+
+  // Equivalent to Set(i) for i in [0, 64) where (bits >> i) & 1. This does
+  // not clear any existing bits.
+  void SetNonzeroBitsFrom64(uint64_t bits) { bits_ |= bits; }
+
+  void Clear(size_t i) {
+    HWY_DASSERT(i < 64);
+    bits_ &= ~(1ULL << i);
+  }
+
+  bool Get(size_t i) const {
+    HWY_DASSERT(i < 64);
+    return (bits_ & (1ULL << i)) != 0;
+  }
+
+  // Returns true if Get(i) would return true for any i in [0, 64).
+  bool Any() const { return bits_ != 0; }
+
+  // Returns true if Get(i) would return true for all i in [0, 64).
+  bool All() const { return bits_ == ~uint64_t{0}; }
+
+  // Returns lowest i such that `Get(i)`. Caller must first ensure `Any()`!
+  size_t First() const {
+    HWY_DASSERT(Any());
+    return Num0BitsBelowLS1Bit_Nonzero64(bits_);
+  }
+
+  // Returns lowest i such that `!Get(i)`. Caller must first ensure `!All()`!
+  size_t First0() const {
+    HWY_DASSERT(!All());
+    return Num0BitsBelowLS1Bit_Nonzero64(~bits_);
+  }
+
+  // Returns uint64_t(Get(i)) << i for i in [0, 64).
+  uint64_t Get64() const { return bits_; }
+
+  // Calls `func(i)` for each `i` in the set. It is safe for `func` to modify
+  // the set, but the current Foreach call is unaffected.
+  template <class Func>
+  void Foreach(const Func& func) const {
+    uint64_t remaining_bits = bits_;
+    while (remaining_bits != 0) {
+      const size_t i = Num0BitsBelowLS1Bit_Nonzero64(remaining_bits);
+      remaining_bits &= remaining_bits - 1;  // clear LSB
+      func(i);
+    }
+  }
+
+  size_t Count() const { return PopCount(bits_); }
+
+ private:
+  uint64_t bits_ = 0;
+};
+
+// Any number of bits, flat array.
+template <size_t kMaxSize>
+class BitSet {
+  static_assert(kMaxSize != 0, "BitSet requires non-zero size");
+
+ public:
+  constexpr size_t MaxSize() const { return kMaxSize; }
+
+  // No harm if `i` is already set.
+  void Set(size_t i) {
+    HWY_DASSERT(i < kMaxSize);
+    const size_t idx = i / 64;
+    const size_t mod = i % 64;
+    bits_[idx].Set(mod);
+  }
+
+  void Clear(size_t i) {
+    HWY_DASSERT(i < kMaxSize);
+    const size_t idx = i / 64;
+    const size_t mod = i % 64;
+    bits_[idx].Clear(mod);
+    HWY_DASSERT(!Get(i));
+  }
+
+  bool Get(size_t i) const {
+    HWY_DASSERT(i < kMaxSize);
+    const size_t idx = i / 64;
+    const size_t mod = i % 64;
+    return bits_[idx].Get(mod);
+  }
+
+  // Returns true if Get(i) would return true for any i in [0, kMaxSize).
+  bool Any() const {
+    for (const BitSet64& bits : bits_) {
+      if (bits.Any()) return true;
+    }
+    return false;
+  }
+
+  // Returns true if Get(i) would return true for all i in [0, kMaxSize).
+  bool All() const {
+    for (size_t idx = 0; idx < kNum64 - 1; ++idx) {
+      if (!bits_[idx].All()) return false;
+    }
+
+    constexpr size_t kRemainder = kMaxSize % 64;
+    if (kRemainder == 0) {
+      return bits_[kNum64 - 1].All();
+    }
+    return bits_[kNum64 - 1].Count() == kRemainder;
+  }
+
+  // Returns lowest i such that `Get(i)`. Caller must first ensure `Any()`!
+  size_t First() const {
+    HWY_DASSERT(Any());
+    for (size_t idx = 0;; ++idx) {
+      HWY_DASSERT(idx < kNum64);
+      if (bits_[idx].Any()) return idx * 64 + bits_[idx].First();
+    }
+  }
+
+  // Returns lowest i such that `!Get(i)`. Caller must first ensure `All()`!
+  size_t First0() const {
+    HWY_DASSERT(!All());
+    for (size_t idx = 0;; ++idx) {
+      HWY_DASSERT(idx < kNum64);
+      if (!bits_[idx].All()) {
+        const size_t first0 = idx * 64 + bits_[idx].First0();
+        HWY_DASSERT(first0 < kMaxSize);
+        return first0;
+      }
+    }
+  }
+
+  // Calls `func(i)` for each `i` in the set. It is safe for `func` to modify
+  // the set, but the current Foreach call is only affected if changing one of
+  // the not yet visited BitSet64.
+  template <class Func>
+  void Foreach(const Func& func) const {
+    for (size_t idx = 0; idx < kNum64; ++idx) {
+      bits_[idx].Foreach([idx, &func](size_t mod) { func(idx * 64 + mod); });
+    }
+  }
+
+  size_t Count() const {
+    size_t total = 0;
+    for (const BitSet64& bits : bits_) {
+      total += bits.Count();
+    }
+    return total;
+  }
+
+ private:
+  static constexpr size_t kNum64 = DivCeil(kMaxSize, size_t{64});
+  BitSet64 bits_[kNum64];
+};
+
+// Any number of bits, flat array, atomic updates to the u64.
+template <size_t kMaxSize>
+class AtomicBitSet {
+  static_assert(kMaxSize != 0, "AtomicBitSet requires non-zero size");
+
+  // Bits may signal something to other threads, hence relaxed is insufficient.
+  // Acq/Rel ensures a happens-before relationship.
+  static constexpr auto kAcq = std::memory_order_acquire;
+  static constexpr auto kRel = std::memory_order_release;
+
+ public:
+  constexpr size_t MaxSize() const { return kMaxSize; }
+
+  // No harm if `i` is already set.
+  void Set(size_t i) {
+    HWY_DASSERT(i < kMaxSize);
+    const size_t idx = i / 64;
+    const size_t mod = i % 64;
+    bits_[idx].fetch_or(1ULL << mod, kRel);
+  }
+
+  void Clear(size_t i) {
+    HWY_DASSERT(i < kMaxSize);
+    const size_t idx = i / 64;
+    const size_t mod = i % 64;
+    bits_[idx].fetch_and(~(1ULL << mod), kRel);
+    HWY_DASSERT(!Get(i));
+  }
+
+  bool Get(size_t i) const {
+    HWY_DASSERT(i < kMaxSize);
+    const size_t idx = i / 64;
+    const size_t mod = i % 64;
+    return ((bits_[idx].load(kAcq) & (1ULL << mod))) != 0;
+  }
+
+  // Returns true if Get(i) would return true for any i in [0, kMaxSize).
+  bool Any() const {
+    for (const std::atomic<uint64_t>& bits : bits_) {
+      if (bits.load(kAcq)) return true;
+    }
+    return false;
+  }
+
+  // Returns true if Get(i) would return true for all i in [0, kMaxSize).
+  bool All() const {
+    for (size_t idx = 0; idx < kNum64 - 1; ++idx) {
+      if (bits_[idx].load(kAcq) != ~uint64_t{0}) return false;
+    }
+
+    constexpr size_t kRemainder = kMaxSize % 64;
+    const uint64_t last_bits = bits_[kNum64 - 1].load(kAcq);
+    if (kRemainder == 0) {
+      return last_bits == ~uint64_t{0};
+    }
+    return PopCount(last_bits) == kRemainder;
+  }
+
+  // Returns lowest i such that `Get(i)`. Caller must first ensure `Any()`!
+  size_t First() const {
+    HWY_DASSERT(Any());
+    for (size_t idx = 0;; ++idx) {
+      HWY_DASSERT(idx < kNum64);
+      const uint64_t bits = bits_[idx].load(kAcq);
+      if (bits != 0) {
+        return idx * 64 + Num0BitsBelowLS1Bit_Nonzero64(bits);
+      }
+    }
+  }
+
+  // Returns lowest i such that `!Get(i)`. Caller must first ensure `!All()`!
+  size_t First0() const {
+    HWY_DASSERT(!All());
+    for (size_t idx = 0;; ++idx) {
+      HWY_DASSERT(idx < kNum64);
+      const uint64_t inv_bits = ~bits_[idx].load(kAcq);
+      if (inv_bits != 0) {
+        const size_t first0 =
+            idx * 64 + Num0BitsBelowLS1Bit_Nonzero64(inv_bits);
+        HWY_DASSERT(first0 < kMaxSize);
+        return first0;
+      }
+    }
+  }
+
+  // Calls `func(i)` for each `i` in the set. It is safe for `func` to modify
+  // the set, but the current Foreach call is only affected if changing one of
+  // the not yet visited uint64_t.
+  template <class Func>
+  void Foreach(const Func& func) const {
+    for (size_t idx = 0; idx < kNum64; ++idx) {
+      uint64_t remaining_bits = bits_[idx].load(kAcq);
+      while (remaining_bits != 0) {
+        const size_t i = Num0BitsBelowLS1Bit_Nonzero64(remaining_bits);
+        remaining_bits &= remaining_bits - 1;  // clear LSB
+        func(idx * 64 + i);
+      }
+    }
+  }
+
+  size_t Count() const {
+    size_t total = 0;
+    for (const std::atomic<uint64_t>& bits : bits_) {
+      total += PopCount(bits.load(kAcq));
+    }
+    return total;
+  }
+
+ private:
+  static constexpr size_t kNum64 = DivCeil(kMaxSize, size_t{64});
+  std::atomic<uint64_t> bits_[kNum64] = {};
+};
+
+// Two-level bitset for up to `kMaxSize` <= 4096 values. The iterators
+// (`Any/First/Foreach/Count`) are more efficient than `BitSet` for sparse sets.
+// This comes at the cost of slightly slower mutators (`Set/Clear`).
+template <size_t kMaxSize = 4096>
+class BitSet4096 {
+  static_assert(kMaxSize != 0, "BitSet4096 requires non-zero size");
+
+ public:
+  constexpr size_t MaxSize() const { return kMaxSize; }
+
+  // No harm if `i` is already set.
+  void Set(size_t i) {
+    HWY_DASSERT(i < kMaxSize);
+    const size_t idx = i / 64;
+    const size_t mod = i % 64;
+    bits_[idx].Set(mod);
+    nonzero_.Set(idx);
+    HWY_DASSERT(Get(i));
+  }
+
+  // Equivalent to Set(i) for i in [0, 64) where (bits >> i) & 1. This does
+  // not clear any existing bits.
+  void SetNonzeroBitsFrom64(uint64_t bits) {
+    bits_[0].SetNonzeroBitsFrom64(bits);
+    if (bits) nonzero_.Set(0);
+  }
+
+  void Clear(size_t i) {
+    HWY_DASSERT(i < kMaxSize);
+    const size_t idx = i / 64;
+    const size_t mod = i % 64;
+    bits_[idx].Clear(mod);
+    if (!bits_[idx].Any()) {
+      nonzero_.Clear(idx);
+    }
+    HWY_DASSERT(!Get(i));
+  }
+
+  bool Get(size_t i) const {
+    HWY_DASSERT(i < kMaxSize);
+    const size_t idx = i / 64;
+    const size_t mod = i % 64;
+    return bits_[idx].Get(mod);
+  }
+
+  // Returns true if `Get(i)` would return true for any i in [0, kMaxSize).
+  bool Any() const { return nonzero_.Any(); }
+
+  // Returns true if `Get(i)` would return true for all i in [0, kMaxSize).
+  bool All() const {
+    // Do not check `nonzero_.All()` - that only works if `kMaxSize` is 4096.
+    if (nonzero_.Count() != kNum64) return false;
+    return Count() == kMaxSize;
+  }
+
+  // Returns lowest i such that `Get(i)`. Caller must first ensure `Any()`!
+  size_t First() const {
+    HWY_DASSERT(Any());
+    const size_t idx = nonzero_.First();
+    return idx * 64 + bits_[idx].First();
+  }
+
+  // Returns lowest i such that `!Get(i)`. Caller must first ensure `!All()`!
+  size_t First0() const {
+    HWY_DASSERT(!All());
+    // It is likely not worthwhile to have a separate `BitSet64` for `not_all_`,
+    // hence iterate over all u64.
+    for (size_t idx = 0;; ++idx) {
+      HWY_DASSERT(idx < kNum64);
+      if (!bits_[idx].All()) {
+        const size_t first0 = idx * 64 + bits_[idx].First0();
+        HWY_DASSERT(first0 < kMaxSize);
+        return first0;
+      }
+    }
+  }
+
+  // Returns uint64_t(Get(i)) << i for i in [0, 64).
+  uint64_t Get64() const { return bits_[0].Get64(); }
+
+  // Calls `func(i)` for each `i` in the set. It is safe for `func` to modify
+  // the set, but the current Foreach call is only affected if changing one of
+  // the not yet visited BitSet64 for which Any() is true.
+  template <class Func>
+  void Foreach(const Func& func) const {
+    nonzero_.Foreach([&func, this](size_t idx) {
+      bits_[idx].Foreach([idx, &func](size_t mod) { func(idx * 64 + mod); });
+    });
+  }
+
+  size_t Count() const {
+    size_t total = 0;
+    nonzero_.Foreach(
+        [&total, this](size_t idx) { total += bits_[idx].Count(); });
+    return total;
+  }
+
+ private:
+  static_assert(kMaxSize <= 64 * 64, "One BitSet64 insufficient");
+  static constexpr size_t kNum64 = DivCeil(kMaxSize, size_t{64});
+  BitSet64 nonzero_;
+  BitSet64 bits_[kNum64];
+};
+
+}  // namespace hwy
+
+#endif  // HIGHWAY_HWY_BIT_SET_H_

package/include/hwy/cache_control.h

@@ -98,9 +98,10 @@ template <typename T>
 HWY_INLINE HWY_ATTR_CACHE void Prefetch(const T* p) {
   (void)p;
 #ifndef HWY_DISABLE_CACHE_CONTROL
-#if HWY_ARCH_X86
+// Use _mm_prefetch on x86/x64, except when clang-cl is compiled with -mno-mmx.
+#if HWY_ARCH_X86 && !(HWY_COMPILER_CLANGCL && !defined(__MMX__))
   _mm_prefetch(reinterpret_cast<const char*>(p), _MM_HINT_T0);
-#elif HWY_COMPILER_GCC  // includes clang
+#elif HWY_COMPILER_GCC || HWY_COMPILER_CLANGCL  // includes clang
   // Hint=0 (NTA) behavior differs, but skipping outer caches is probably not
   // desirable, so use the default 3 (keep in caches).
   __builtin_prefetch(p, /*write=*/0, /*hint=*/3);