numkong 7.4.5 → 7.6.0

package/include/numkong/mesh.h

@@ -266,6 +266,20 @@ NK_PUBLIC void nk_umeyama_bf16_skylake(nk_bf16_t const *a, nk_bf16_t const *b, n
                                        nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result);
 #endif // NK_TARGET_SKYLAKE
 
+/*  SIMD-powered backends for AVX512-BF16 CPUs of AMD Genoa / Intel Sapphire Rapids generation and newer.
+ */
+#if NK_TARGET_GENOA
+/** @copydoc nk_rmsd_bf16 */
+NK_PUBLIC void nk_rmsd_bf16_genoa(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
+                                  nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result);
+/** @copydoc nk_kabsch_bf16 */
+NK_PUBLIC void nk_kabsch_bf16_genoa(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
+                                    nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result);
+/** @copydoc nk_umeyama_bf16 */
+NK_PUBLIC void nk_umeyama_bf16_genoa(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
+                                     nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result);
+#endif // NK_TARGET_GENOA
+
 /*  SIMD-powered backends for AVX2 CPUs of Haswell generation and newer.
  */
 #if NK_TARGET_HASWELL
@@ -357,6 +371,20 @@ NK_PUBLIC void nk_umeyama_bf16_neonbfdot(nk_bf16_t const *a, nk_bf16_t const *b,
                                          nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result);
 #endif // NK_TARGET_NEONBFDOT
 
+/*  SIMD-powered backends for Arm NEON FHM (FP16 widening FMA) CPUs.
+ */
+#if NK_TARGET_NEONFHM
+/** @copydoc nk_rmsd_f16 */
+NK_PUBLIC void nk_rmsd_f16_neonfhm(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
+                                   nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result);
+/** @copydoc nk_kabsch_f16 */
+NK_PUBLIC void nk_kabsch_f16_neonfhm(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
+                                     nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result);
+/** @copydoc nk_umeyama_f16 */
+NK_PUBLIC void nk_umeyama_f16_neonfhm(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
+                                      nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result);
+#endif // NK_TARGET_NEONFHM
+
 #if NK_TARGET_RVV
 /** @copydoc nk_rmsd_f32 */
 NK_PUBLIC void nk_rmsd_f32_rvv(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t *a_centroid,
@@ -454,8 +482,10 @@ NK_INTERNAL nk_dtype_t nk_mesh_transform_dtype(nk_dtype_t dtype) {
 #include "numkong/mesh/serial.h"
 #include "numkong/mesh/neon.h"
 #include "numkong/mesh/neonbfdot.h"
+#include "numkong/mesh/neonfhm.h"
 #include "numkong/mesh/haswell.h"
 #include "numkong/mesh/skylake.h"
+#include "numkong/mesh/genoa.h"
 #include "numkong/mesh/rvv.h"
 #include "numkong/mesh/v128relaxed.h"
 
@@ -505,6 +535,8 @@ NK_PUBLIC void nk_rmsd_f16(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk
     nk_rmsd_f16_skylake(a, b, n, a_centroid, b_centroid, rotation, scale, result);
 #elif NK_TARGET_HASWELL
     nk_rmsd_f16_haswell(a, b, n, a_centroid, b_centroid, rotation, scale, result);
+#elif NK_TARGET_NEONFHM
+    nk_rmsd_f16_neonfhm(a, b, n, a_centroid, b_centroid, rotation, scale, result);
 #elif NK_TARGET_NEON
     nk_rmsd_f16_neon(a, b, n, a_centroid, b_centroid, rotation, scale, result);
 #elif NK_TARGET_RVV
@@ -517,6 +549,8 @@ NK_PUBLIC void nk_rmsd_f16(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk
 NK_PUBLIC void nk_rmsd_bf16(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t *a_centroid,
                             nk_f32_t *b_centroid, nk_f32_t *rotation, nk_f32_t *scale, nk_f32_t *result) {
 #if NK_TARGET_SKYLAKE
+    //  Skylake f32-widen path wins on Intel where VDPBF16PS throughput matches FMA; on AMD Zen4+
+    //  where VDPBF16PS is faster than FMA, users can call `nk_rmsd_bf16_genoa` directly.
     nk_rmsd_bf16_skylake(a, b, n, a_centroid, b_centroid, rotation, scale, result);
 #elif NK_TARGET_HASWELL
     nk_rmsd_bf16_haswell(a, b, n, a_centroid, b_centroid, rotation, scale, result);
@@ -569,6 +603,8 @@ NK_PUBLIC void nk_kabsch_f16(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n,
     nk_kabsch_f16_skylake(a, b, n, a_centroid, b_centroid, rotation, scale, result);
 #elif NK_TARGET_HASWELL
     nk_kabsch_f16_haswell(a, b, n, a_centroid, b_centroid, rotation, scale, result);
+#elif NK_TARGET_NEONFHM
+    nk_kabsch_f16_neonfhm(a, b, n, a_centroid, b_centroid, rotation, scale, result);
 #elif NK_TARGET_NEON
     nk_kabsch_f16_neon(a, b, n, a_centroid, b_centroid, rotation, scale, result);
 #elif NK_TARGET_RVV
@@ -633,6 +669,8 @@ NK_PUBLIC void nk_umeyama_f16(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n,
     nk_umeyama_f16_skylake(a, b, n, a_centroid, b_centroid, rotation, scale, result);
 #elif NK_TARGET_HASWELL
     nk_umeyama_f16_haswell(a, b, n, a_centroid, b_centroid, rotation, scale, result);
+#elif NK_TARGET_NEONFHM
+    nk_umeyama_f16_neonfhm(a, b, n, a_centroid, b_centroid, rotation, scale, result);
 #elif NK_TARGET_NEON
     nk_umeyama_f16_neon(a, b, n, a_centroid, b_centroid, rotation, scale, result);
 #elif NK_TARGET_RVV

package/include/numkong/reduce/neon.h

@@ -3936,6 +3936,35 @@ NK_PUBLIC void nk_reduce_moments_f16_neon(                             //
     else nk_reduce_moments_f16_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr);
 }
 
+NK_INTERNAL void nk_reduce_moments_u1_neon_contiguous_( //
+    nk_u1x8_t const *data_ptr, nk_size_t count,         //
+    nk_u64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
+    nk_size_t byte_count = nk_size_divide_round_up_(count, NK_BITS_PER_BYTE);
+    nk_u64_t sum = 0;
+    nk_size_t idx = 0;
+    // Each vcntq_u8 produces values 0-8 per lane; accumulate at u8 level
+    // for up to 31 iterations (31 × 8 = 248, fits in u8) before widening.
+    while (idx + 16 <= byte_count) {
+        uint8x16_t popcount_u8x16 = vdupq_n_u8(0);
+        for (nk_size_t cycle = 0; cycle < 31 && idx + 16 <= byte_count; ++cycle, idx += 16) {
+            uint8x16_t data_u8x16 = vld1q_u8((nk_u8_t const *)data_ptr + idx);
+            popcount_u8x16 = vaddq_u8(popcount_u8x16, vcntq_u8(data_u8x16));
+        }
+        sum += (nk_u64_t)vaddlvq_u8(popcount_u8x16);
+    }
+    for (; idx < byte_count; ++idx) sum += nk_u1x8_popcount_(((nk_u8_t const *)data_ptr)[idx]);
+    *sum_ptr = sum, *sumsq_ptr = sum;
+}
+
+NK_PUBLIC void nk_reduce_moments_u1_neon(                               //
+    nk_u1x8_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
+    nk_u64_t *sum_ptr, nk_u64_t *sumsq_ptr) {
+    count = nk_size_round_up_to_multiple_(count, 8);
+    if (count == 0) *sum_ptr = 0, *sumsq_ptr = 0;
+    else if (stride_bytes == 1) nk_reduce_moments_u1_neon_contiguous_(data_ptr, count, sum_ptr, sumsq_ptr);
+    else nk_reduce_moments_u1_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr);
+}
+
 #if defined(__clang__)
 #pragma clang attribute pop
 #elif defined(__GNUC__)

package/include/numkong/reduce/neonbfdot.h

@@ -33,7 +33,7 @@ NK_INTERNAL void nk_reduce_moments_bf16_neonbfdot_contiguous_( //
     nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
 
     // bf16 representation of 1.0 is 0x3F80 (same as upper 16 bits of f32 1.0)
-    bfloat16x8_t ones_bf16x8 = vreinterpretq_bf16_u16(vdupq_n_u16(0x3F80));
+    bfloat16x8_t ones_bf16x8 = vreinterpretq_bf16_u16(nk_u16x8_splat_(0x3F80));
     float32x4_t sum_f32x4 = vdupq_n_f32(0);
     float32x4_t sumsq_f32x4 = vdupq_n_f32(0);
     nk_size_t idx = 0;
@@ -61,7 +61,7 @@ NK_INTERNAL void nk_reduce_moments_bf16_neonbfdot_strided_(                //
     nk_bf16_t const *data_ptr, nk_size_t count, nk_size_t stride_elements, //
     nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
 
-    bfloat16x8_t ones_bf16x8 = vreinterpretq_bf16_u16(vdupq_n_u16(0x3F80));
+    bfloat16x8_t ones_bf16x8 = vreinterpretq_bf16_u16(nk_u16x8_splat_(0x3F80));
     float32x4_t sum_f32x4 = vdupq_n_f32(0);
     float32x4_t sumsq_f32x4 = vdupq_n_f32(0);
     nk_size_t idx = 0;

package/include/numkong/reduce/neonfhm.h

@@ -34,7 +34,7 @@ NK_INTERNAL void nk_reduce_moments_e4m3_neonfhm_contiguous_( //
 
     float32x4_t sum_f32x4 = vdupq_n_f32(0);
     float32x4_t sumsq_f32x4 = vdupq_n_f32(0);
-    float16x8_t ones_f16x8 = vreinterpretq_f16_u16(vdupq_n_u16(0x3C00));
+    float16x8_t ones_f16x8 = vreinterpretq_f16_u16(nk_u16x8_splat_(0x3C00));
     nk_size_t idx = 0;
 
     for (; idx + 8 <= count; idx += 8) {
@@ -67,7 +67,7 @@ NK_INTERNAL void nk_reduce_moments_e4m3_neonfhm_strided_(                  //
 
     float32x4_t sum_f32x4 = vdupq_n_f32(0);
     float32x4_t sumsq_f32x4 = vdupq_n_f32(0);
-    float16x8_t ones_f16x8 = vreinterpretq_f16_u16(vdupq_n_u16(0x3C00));
+    float16x8_t ones_f16x8 = vreinterpretq_f16_u16(nk_u16x8_splat_(0x3C00));
     nk_size_t idx = 0;
 
     if (stride_elements == 2) {
@@ -159,7 +159,7 @@ NK_INTERNAL void nk_reduce_moments_e5m2_neonfhm_contiguous_( //
 
     float32x4_t sum_f32x4 = vdupq_n_f32(0);
     float32x4_t sumsq_f32x4 = vdupq_n_f32(0);
-    float16x8_t ones_f16x8 = vreinterpretq_f16_u16(vdupq_n_u16(0x3C00));
+    float16x8_t ones_f16x8 = vreinterpretq_f16_u16(nk_u16x8_splat_(0x3C00));
     nk_size_t idx = 0;
 
     for (; idx + 8 <= count; idx += 8) {
@@ -192,7 +192,7 @@ NK_INTERNAL void nk_reduce_moments_e5m2_neonfhm_strided_(                  //
 
     float32x4_t sum_f32x4 = vdupq_n_f32(0);
     float32x4_t sumsq_f32x4 = vdupq_n_f32(0);
-    float16x8_t ones_f16x8 = vreinterpretq_f16_u16(vdupq_n_u16(0x3C00));
+    float16x8_t ones_f16x8 = vreinterpretq_f16_u16(nk_u16x8_splat_(0x3C00));
     nk_size_t idx = 0;
 
     if (stride_elements == 2) {

package/include/numkong/reduce/serial.h

@@ -14,7 +14,6 @@
 #define NK_REDUCE_SERIAL_H
 
 #include "numkong/types.h"
-#include "numkong/scalar/serial.h"
 #include "numkong/cast/serial.h"
 #include "numkong/scalar/serial.h"
 
@@ -22,6 +21,15 @@
 extern "C" {
 #endif
 
+/*  Keep the serial instantiations below actually scalar, regardless of build type.
+ *  See dots/serial.h for rationale. */
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
 NK_INTERNAL nk_f64_t nk_reduce_sum_f64_serial_(nk_f64_t const *values, nk_f64_t const *compensations, int count) {
     nk_f64_t running_sum = 0, accumulated_error = 0;
     for (int i = 0; i < count; i++) {
@@ -746,6 +754,12 @@ NK_PUBLIC void nk_reduce_minmax_u1_serial(                          //
     *max_value_ptr = max_value, *max_index_ptr = max_idx;
 }
 
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
 #if defined(__cplusplus)
 } // extern "C"
 #endif

package/include/numkong/reduce/sve.h

@@ -0,0 +1,52 @@
+/**
+ *  @brief SVE horizontal reduction helpers with MSan unpoisoning.
+ *  @file include/numkong/reduce/sve.h
+ *  @author Ash Vardanian
+ *  @date April 12, 2026
+ *
+ *  LLVM's MSan does not instrument ARM SVE intrinsics — `svaddv` moves data
+ *  from vector to scalar registers via architecture-specific paths invisible
+ *  to the compiler, causing false-positive uninitialized-value reports.
+ *  These macros wrap the reduction and unpoison the scalar result.
+ *
+ *  The `svaddv` intrinsic stays inside a macro so it expands in the caller's
+ *  target context — SVE and SME streaming translation units carry incompatible
+ *  target attributes. The unpoisoning runs on the already-reduced scalar, so it
+ *  lives in a target-agnostic `NK_INTERNAL` helper called from the macro.
+ *
+ *  @sa include/numkong/reduce.h
+ */
+#ifndef NK_REDUCE_SVE_H
+#define NK_REDUCE_SVE_H
+
+#if NK_TARGET_ARM64_
+#if NK_TARGET_SVE || NK_TARGET_SVE2 || NK_TARGET_SME
+
+#include "numkong/types.h"
+
+NK_INTERNAL nk_f64_t nk_unpoison_f64_(nk_f64_t v) NK_STREAMING_COMPATIBLE_ {
+    nk_unpoison_(&v, sizeof(v));
+    return v;
+}
+NK_INTERNAL nk_f32_t nk_unpoison_f32_(nk_f32_t v) NK_STREAMING_COMPATIBLE_ {
+    nk_unpoison_(&v, sizeof(v));
+    return v;
+}
+NK_INTERNAL nk_u64_t nk_unpoison_u64_(nk_u64_t v) NK_STREAMING_COMPATIBLE_ {
+    nk_unpoison_(&v, sizeof(v));
+    return v;
+}
+NK_INTERNAL nk_i64_t nk_unpoison_i64_(nk_i64_t v) NK_STREAMING_COMPATIBLE_ {
+    nk_unpoison_(&v, sizeof(v));
+    return v;
+}
+
+#define nk_svaddv_f64_(predicate, vector) nk_unpoison_f64_(svaddv_f64((predicate), (vector)))
+#define nk_svaddv_f32_(predicate, vector) nk_unpoison_f32_(svaddv_f32((predicate), (vector)))
+#define nk_svaddv_u32_(predicate, vector) nk_unpoison_u64_(svaddv_u32((predicate), (vector)))
+#define nk_svaddv_s32_(predicate, vector) nk_unpoison_i64_(svaddv_s32((predicate), (vector)))
+#define nk_svaddv_u8_(predicate, vector)  nk_unpoison_u64_(svaddv_u8((predicate), (vector)))
+
+#endif // NK_TARGET_SVE || NK_TARGET_SVE2 || NK_TARGET_SME
+#endif // NK_TARGET_ARM64_
+#endif // NK_REDUCE_SVE_H

package/include/numkong/reduce.h

@@ -389,6 +389,8 @@ NK_PUBLIC void nk_reduce_moments_i16_neon(nk_i16_t const *, nk_size_t, nk_size_t
 /** @copydoc nk_reduce_moments_f64 */
 NK_PUBLIC void nk_reduce_moments_u16_neon(nk_u16_t const *, nk_size_t, nk_size_t, nk_u64_t *, nk_u64_t *);
 /** @copydoc nk_reduce_moments_f64 */
+NK_PUBLIC void nk_reduce_moments_u1_neon(nk_u1x8_t const *, nk_size_t, nk_size_t, nk_u64_t *, nk_u64_t *);
+/** @copydoc nk_reduce_moments_f64 */
 NK_PUBLIC void nk_reduce_moments_i32_neon(nk_i32_t const *, nk_size_t, nk_size_t, nk_i64_t *, nk_u64_t *);
 /** @copydoc nk_reduce_moments_f64 */
 NK_PUBLIC void nk_reduce_moments_u32_neon(nk_u32_t const *, nk_size_t, nk_size_t, nk_u64_t *, nk_u64_t *);
@@ -1559,6 +1561,8 @@ NK_PUBLIC void nk_reduce_moments_u1(nk_u1x8_t const *d, nk_size_t n, nk_size_t s
     nk_reduce_moments_u1_skylake(d, n, s, sum, sumsq);
 #elif NK_TARGET_HASWELL
     nk_reduce_moments_u1_haswell(d, n, s, sum, sumsq);
+#elif NK_TARGET_NEON
+    nk_reduce_moments_u1_neon(d, n, s, sum, sumsq);
 #else
     nk_reduce_moments_u1_serial(d, n, s, sum, sumsq);
 #endif

package/include/numkong/set/sve.h

@@ -32,8 +32,9 @@
 #if NK_TARGET_ARM64_
 #if NK_TARGET_SVE
 
-#include "numkong/types.h"    // `nk_u1x8_t`
-#include "numkong/set/neon.h" // `nk_hamming_u1_neon`
+#include "numkong/types.h"      // `nk_u1x8_t`
+#include "numkong/reduce/sve.h" // `nk_svaddv_f64_`
+#include "numkong/set/neon.h"   // `nk_hamming_u1_neon`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -73,7 +74,7 @@ NK_PUBLIC void nk_hamming_u1_sve(nk_u1x8_t const *a, nk_u1x8_t const *b, nk_size
             i += words_per_register;
             ++cycle;
         } while (i < n_bytes && cycle < 31);
-        differences += svaddv_u8(all_predicate_b8x, popcount_u8x);
+        differences += nk_svaddv_u8_(all_predicate_b8x, popcount_u8x);
         popcount_u8x = svdup_n_u8(0);
         cycle = 0; // Reset the cycle counter.
     }
@@ -110,9 +111,9 @@ NK_PUBLIC void nk_jaccard_u1_sve(nk_u1x8_t const *a, nk_u1x8_t const *b, nk_size
             i += words_per_register;
             ++cycle;
         } while (i < n_bytes && cycle < 31);
-        intersection_count += svaddv_u8(all_predicate_b8x, intersection_popcount_u8x);
+        intersection_count += nk_svaddv_u8_(all_predicate_b8x, intersection_popcount_u8x);
         intersection_popcount_u8x = svdup_n_u8(0);
-        union_count += svaddv_u8(all_predicate_b8x, union_popcount_u8x);
+        union_count += nk_svaddv_u8_(all_predicate_b8x, union_popcount_u8x);
         union_popcount_u8x = svdup_n_u8(0);
         cycle = 0; // Reset the cycle counter.
     }

package/include/numkong/sets/smebi32.h

@@ -41,8 +41,9 @@
 #include "numkong/types.h"
 #include "numkong/set/serial.h"
 #include "numkong/sets/serial.h"
-#include "numkong/dots/sme.h" // `nk_sme_zero_za32_*` constants
-#include "numkong/reduce.h"   // `nk_reduce_moments_u1`
+#include "numkong/reduce/sve.h"  // `nk_svaddv_f64_`
+#include "numkong/reduce/neon.h" // `nk_reduce_moments_u1_neon`
+#include "numkong/dots/sme.h"    // `nk_sme_zero_za32_*`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -100,7 +101,7 @@ NK_PUBLIC nk_u32_t nk_sets_reduce_sumsq_u1_streaming_(nk_u1x8_t const *data, nk_
         svbool_t predicate_b8x = svwhilelt_b8_u64(offset, n_bytes);
         acc_u32x = svdot_u32(acc_u32x, svcnt_u8_z(predicate_b8x, svld1_u8(predicate_b8x, data + offset)), ones_u8x);
     }
-    return (nk_u32_t)svaddv_u32(svptrue_b32(), acc_u32x);
+    return (nk_u32_t)nk_svaddv_u32_(svptrue_b32(), acc_u32x);
 }
 
 #pragma region Hamming Distance
@@ -187,11 +188,9 @@ NK_PUBLIC void nk_dots_pack_u1_smebi32(nk_u1x8_t const *b, nk_size_t row_count,
     // Compute per-row population counts
     for (nk_size_t row = 0; row < row_count; row++) {
         nk_u1x8_t const *src_row = (nk_u1x8_t const *)((char const *)b + row * b_stride_in_bytes);
-        {
-            nk_u64_t nk_local_sum_, nk_local_sumsq_;
-            nk_reduce_moments_u1(src_row, depth_bytes * 8, sizeof(nk_u1x8_t), &nk_local_sum_, &nk_local_sumsq_);
-            norms_ptr[row] = (nk_u32_t)nk_local_sum_;
-        }
+        nk_u64_t nk_local_sum_, nk_local_sumsq_;
+        nk_reduce_moments_u1_neon(src_row, depth_bytes * 8, sizeof(nk_u1x8_t), &nk_local_sum_, &nk_local_sumsq_);
+        norms_ptr[row] = (nk_u32_t)nk_local_sum_;
     }
 }
 
@@ -203,9 +202,9 @@ NK_PUBLIC void nk_dots_pack_u1_smebi32(nk_u1x8_t const *b, nk_size_t row_count,
  *  Each ZA0.S batch covers 16 depth u32 steps (one full depth tile).
  *  BMOPA expansion=1 for u32: each u32 contributes 32 bits via XNOR+POPCNT.
  */
-__arm_locally_streaming __arm_new("za") static void nk_hammings_packed_u1_smebi32_streaming_(
+__arm_new("za") static void nk_hammings_packed_u1_smebi32_streaming_( //
     nk_u1x8_t const *a, void const *b_packed, nk_u32_t *c, nk_size_t row_count_a, nk_size_t row_count_b,
-    nk_size_t depth_bits, nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+    nk_size_t depth_bits, nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) NK_STREAMING_ {
 
     nk_sets_smebi32_packed_header_t const *header = (nk_sets_smebi32_packed_header_t const *)b_packed;
     nk_size_t const row_tile_count_b = header->row_tile_count;
@@ -344,11 +343,13 @@ __arm_locally_streaming __arm_new("za") static void nk_hammings_packed_u1_smebi3
     }
 }
 
-NK_PUBLIC void nk_hammings_packed_u1_smebi32(nk_u1x8_t const *a, void const *b_packed, nk_u32_t *c,
-                                             nk_size_t row_count_a, nk_size_t row_count_b, nk_size_t depth_bits,
-                                             nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+NK_PUBLIC void nk_hammings_packed_u1_smebi32( //
+    nk_u1x8_t const *a, void const *b_packed, nk_u32_t *c, nk_size_t row_count_a, nk_size_t row_count_b,
+    nk_size_t depth_bits, nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+    nk_sme_start_streaming_();
     nk_hammings_packed_u1_smebi32_streaming_(a, b_packed, c, row_count_a, row_count_b, depth_bits, a_stride_in_bytes,
                                              c_stride_in_bytes);
+    nk_sme_stop_streaming_();
 }
 
 /**
@@ -357,9 +358,9 @@ NK_PUBLIC void nk_hammings_packed_u1_smebi32(nk_u1x8_t const *a, void const *b_p
  *  ZA1-3.S = BMOPA accumulators (3 B column tiles in fast path).
  *  Mirrors the unpacked kernel nk_hammings_packed_u1_smebi32_streaming_ pattern.
  */
-__arm_locally_streaming __arm_new("za") static void nk_hammings_symmetric_u1_smebi32_streaming_(
+__arm_new("za") static void nk_hammings_symmetric_u1_smebi32_streaming_( //
     nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t depth_bits, nk_size_t stride_in_bytes,
-    nk_u32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+    nk_u32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) NK_STREAMING_ {
 
     nk_size_t const tile_dim = svcntw();        // 16 for 512-bit SVL
     nk_size_t const depth_tile_size = svcntw(); // 16 u32 per depth tile
@@ -545,12 +546,13 @@ __arm_locally_streaming __arm_new("za") static void nk_hammings_symmetric_u1_sme
     }
 }
 
-NK_PUBLIC void nk_hammings_symmetric_u1_smebi32(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t depth_bits,
-                                                nk_size_t stride_in_bytes, nk_u32_t *result,
-                                                nk_size_t result_stride_in_bytes, nk_size_t row_start,
-                                                nk_size_t row_count) {
+NK_PUBLIC void nk_hammings_symmetric_u1_smebi32( //
+    nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t depth_bits, nk_size_t stride_in_bytes,
+    nk_u32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+    nk_sme_start_streaming_();
     nk_hammings_symmetric_u1_smebi32_streaming_(vectors, vectors_count, depth_bits, stride_in_bytes, result,
                                                 result_stride_in_bytes, row_start, row_count);
+    nk_sme_stop_streaming_();
 }
 
 #pragma endregion Hamming Distance
@@ -581,9 +583,9 @@ NK_PUBLIC void nk_hammings_symmetric_u1_smebi32(nk_u1x8_t const *vectors, nk_siz
  *    union         = (norm_a + norm_b + hamming) / 2
  *    jaccard       = 1 - intersection / union      (1.0 when union == 0)
  */
-__arm_locally_streaming __arm_new("za") static void nk_jaccards_packed_u1_smebi32_streaming_(
+__arm_new("za") static void nk_jaccards_packed_u1_smebi32_streaming_( //
     nk_u1x8_t const *a, void const *b_packed, nk_f32_t *c, nk_size_t row_count_a, nk_size_t row_count_b,
-    nk_size_t depth_bits, nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+    nk_size_t depth_bits, nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) NK_STREAMING_ {
 
     nk_sets_smebi32_packed_header_t const *header = (nk_sets_smebi32_packed_header_t const *)b_packed;
     nk_size_t const row_tile_count_b = header->row_tile_count;
@@ -796,11 +798,13 @@ __arm_locally_streaming __arm_new("za") static void nk_jaccards_packed_u1_smebi3
     }
 }
 
-NK_PUBLIC void nk_jaccards_packed_u1_smebi32(nk_u1x8_t const *a, void const *b_packed, nk_f32_t *c,
-                                             nk_size_t row_count_a, nk_size_t row_count_b, nk_size_t depth_bits,
-                                             nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+NK_PUBLIC void nk_jaccards_packed_u1_smebi32( //
+    nk_u1x8_t const *a, void const *b_packed, nk_f32_t *c, nk_size_t row_count_a, nk_size_t row_count_b,
+    nk_size_t depth_bits, nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+    nk_sme_start_streaming_();
     nk_jaccards_packed_u1_smebi32_streaming_(a, b_packed, c, row_count_a, row_count_b, depth_bits, a_stride_in_bytes,
                                              c_stride_in_bytes);
+    nk_sme_stop_streaming_();
 }
 
 /**
@@ -808,9 +812,9 @@ NK_PUBLIC void nk_jaccards_packed_u1_smebi32(nk_u1x8_t const *a, void const *b_p
  *  Fills upper triangle only (column_tile >= row_tile); caller sees result[i][j] for j >= i.
  *  Norms computed on-the-fly using streaming SVE popcount.
  */
-__arm_locally_streaming __arm_new("za") static void nk_jaccards_symmetric_u1_smebi32_streaming_(
+__arm_new("za") static void nk_jaccards_symmetric_u1_smebi32_streaming_( //
     nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t depth_bits, nk_size_t stride_in_bytes,
-    nk_f32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+    nk_f32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) NK_STREAMING_ {
 
     nk_size_t const tile_dim = svcntw();        // 16 for 512-bit SVL
     nk_size_t const depth_tile_size = svcntw(); // 16 u32 per depth tile
@@ -1104,12 +1108,13 @@ __arm_locally_streaming __arm_new("za") static void nk_jaccards_symmetric_u1_sme
     }
 }
 
-NK_PUBLIC void nk_jaccards_symmetric_u1_smebi32(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t depth_bits,
-                                                nk_size_t stride_in_bytes, nk_f32_t *result,
-                                                nk_size_t result_stride_in_bytes, nk_size_t row_start,
-                                                nk_size_t row_count) {
+NK_PUBLIC void nk_jaccards_symmetric_u1_smebi32( //
+    nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t depth_bits, nk_size_t stride_in_bytes,
+    nk_f32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+    nk_sme_start_streaming_();
     nk_jaccards_symmetric_u1_smebi32_streaming_(vectors, vectors_count, depth_bits, stride_in_bytes, result,
                                                 result_stride_in_bytes, row_start, row_count);
+    nk_sme_stop_streaming_();
 }
 
 #pragma endregion Jaccard Distance

package/include/numkong/sparse/serial.h

@@ -17,7 +17,7 @@ extern "C" {
 #endif
 
 #define nk_define_sparse_intersect_(input_type)                                                                      \
-    NK_PUBLIC nk_size_t nk_sparse_intersect_##input_type##_galloping_search_(                                        \
+    NK_INTERNAL nk_size_t nk_sparse_intersect_##input_type##_galloping_search_(                                      \
         nk_##input_type##_t const *array, nk_size_t start, nk_size_t length, nk_##input_type##_t val) {              \
         nk_size_t low = start;                                                                                       \
         nk_size_t high = start + 1;                                                                                  \
@@ -32,7 +32,7 @@ extern "C" {
         }                                                                                                            \
         return low;                                                                                                  \
     }                                                                                                                \
-    NK_PUBLIC nk_size_t nk_sparse_intersect_##input_type##_linear_scan_(                                             \
+    NK_INTERNAL nk_size_t nk_sparse_intersect_##input_type##_linear_scan_(                                           \
         nk_##input_type##_t const *a, nk_##input_type##_t const *b, nk_size_t a_length, nk_size_t b_length,          \
         nk_##input_type##_t *result) {                                                                               \
         nk_size_t intersection_size = 0;                                                                             \
@@ -103,6 +103,15 @@ extern "C" {
         *product = weights_product;                                                                        \
     }
 
+/*  Keep the serial instantiations below actually scalar, regardless of build type.
+ *  See dots/serial.h for rationale. */
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
 nk_define_sparse_intersect_(u16) // nk_sparse_intersect_u16_serial
 nk_define_sparse_intersect_(u32) // nk_sparse_intersect_u32_serial
 nk_define_sparse_intersect_(u64) // nk_sparse_intersect_u64_serial
@@ -110,6 +119,12 @@ nk_define_sparse_intersect_(u64) // nk_sparse_intersect_u64_serial
 nk_define_sparse_dot_(u16, bf16, f32, nk_bf16_to_f32_serial) // nk_sparse_dot_u16bf16_serial
 nk_define_sparse_dot_(u32, f32, f64, nk_assign_from_to_)     // nk_sparse_dot_u32f32_serial
 
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
 #if defined(__cplusplus)
 } // extern "C"
 #endif

package/include/numkong/sparse/sve2.h

@@ -12,6 +12,7 @@
 #if NK_TARGET_ARM64_
 
 #include "numkong/types.h"
+#include "numkong/reduce/sve.h" // `nk_svaddv_f64_`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -395,7 +396,7 @@ NK_PUBLIC void nk_sparse_dot_u32f32_sve2(                 //
         a_idx += a_step;
         b_idx += b_step;
     }
-    *product = svaddv_f64(predicate_all_b64x, product_f64x);
+    *product = nk_svaddv_f64_(predicate_all_b64x, product_f64x);
 }
 
 #if defined(__clang__)
@@ -485,7 +486,7 @@ NK_PUBLIC void nk_sparse_dot_u16bf16_sve2(                  //
         a_idx += a_step;
         b_idx += b_step;
     }
-    *product = svaddv_f32(svptrue_b32(), product_f32x);
+    *product = nk_svaddv_f32_(svptrue_b32(), product_f32x);
 }
 
 #if defined(__clang__)

package/include/numkong/spatial/genoa.h

@@ -139,74 +139,6 @@ nk_angular_bf16_genoa_cycle:
     *result = nk_angular_normalize_f32_haswell_(dot_product_f32, a_norm_sq_f32, b_norm_sq_f32);
 }
 
-NK_PUBLIC void nk_sqeuclidean_e5m2_genoa(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_f32_t *result) {
-    __m512 a_sq_f32x16 = _mm512_setzero_ps();
-    __m512 b_sq_f32x16 = _mm512_setzero_ps();
-    __m512 ab_f32x16 = _mm512_setzero_ps();
-    __m256i a_e5m2x32, b_e5m2x32;
-
-nk_sqeuclidean_e5m2_genoa_cycle:
-    if (n < 32) {
-        __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
-        a_e5m2x32 = _mm256_maskz_loadu_epi8(mask, a);
-        b_e5m2x32 = _mm256_maskz_loadu_epi8(mask, b);
-        n = 0;
-    }
-    else {
-        a_e5m2x32 = _mm256_loadu_epi8(a);
-        b_e5m2x32 = _mm256_loadu_epi8(b);
-        a += 32, b += 32, n -= 32;
-    }
-    __m512i a_bf16x32 = nk_e5m2x32_to_bf16x32_icelake_(a_e5m2x32);
-    __m512i b_bf16x32 = nk_e5m2x32_to_bf16x32_icelake_(b_e5m2x32);
-    a_sq_f32x16 = _mm512_dpbf16_ps(a_sq_f32x16, nk_m512bh_from_m512i_(a_bf16x32), nk_m512bh_from_m512i_(a_bf16x32));
-    b_sq_f32x16 = _mm512_dpbf16_ps(b_sq_f32x16, nk_m512bh_from_m512i_(b_bf16x32), nk_m512bh_from_m512i_(b_bf16x32));
-    ab_f32x16 = _mm512_dpbf16_ps(ab_f32x16, nk_m512bh_from_m512i_(a_bf16x32), nk_m512bh_from_m512i_(b_bf16x32));
-    if (n) goto nk_sqeuclidean_e5m2_genoa_cycle;
-
-    // (a-b)² = a² + b² - 2ab
-    __m512 sum_sq_f32x16 = _mm512_add_ps(a_sq_f32x16, b_sq_f32x16);
-    *result = nk_reduce_add_f32x16_skylake_(_mm512_fnmadd_ps(_mm512_set1_ps(2.0f), ab_f32x16, sum_sq_f32x16));
-}
-
-NK_PUBLIC void nk_euclidean_e5m2_genoa(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_f32_t *result) {
-    nk_sqeuclidean_e5m2_genoa(a, b, n, result);
-    *result = nk_f32_sqrt_haswell(*result);
-}
-
-NK_PUBLIC void nk_angular_e5m2_genoa(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_f32_t *result) {
-    __m512 dot_f32x16 = _mm512_setzero_ps();
-    __m512 a_norm_sq_f32x16 = _mm512_setzero_ps();
-    __m512 b_norm_sq_f32x16 = _mm512_setzero_ps();
-    __m256i a_e5m2x32, b_e5m2x32;
-
-nk_angular_e5m2_genoa_cycle:
-    if (n < 32) {
-        __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
-        a_e5m2x32 = _mm256_maskz_loadu_epi8(mask, a);
-        b_e5m2x32 = _mm256_maskz_loadu_epi8(mask, b);
-        n = 0;
-    }
-    else {
-        a_e5m2x32 = _mm256_loadu_epi8(a);
-        b_e5m2x32 = _mm256_loadu_epi8(b);
-        a += 32, b += 32, n -= 32;
-    }
-    __m512i a_bf16x32 = nk_e5m2x32_to_bf16x32_icelake_(a_e5m2x32);
-    __m512i b_bf16x32 = nk_e5m2x32_to_bf16x32_icelake_(b_e5m2x32);
-    dot_f32x16 = _mm512_dpbf16_ps(dot_f32x16, nk_m512bh_from_m512i_(a_bf16x32), nk_m512bh_from_m512i_(b_bf16x32));
-    a_norm_sq_f32x16 = _mm512_dpbf16_ps(a_norm_sq_f32x16, nk_m512bh_from_m512i_(a_bf16x32),
-                                        nk_m512bh_from_m512i_(a_bf16x32));
-    b_norm_sq_f32x16 = _mm512_dpbf16_ps(b_norm_sq_f32x16, nk_m512bh_from_m512i_(b_bf16x32),
-                                        nk_m512bh_from_m512i_(b_bf16x32));
-    if (n) goto nk_angular_e5m2_genoa_cycle;
-
-    nk_f32_t dot_f32 = nk_reduce_add_f32x16_skylake_(dot_f32x16);
-    nk_f32_t a_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(a_norm_sq_f32x16);
-    nk_f32_t b_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(b_norm_sq_f32x16);
-    *result = nk_angular_normalize_f32_haswell_(dot_f32, a_norm_sq_f32, b_norm_sq_f32);
-}
-
 #if defined(__clang__)
 #pragma clang attribute pop
 #elif defined(__GNUC__)