numkong 7.4.5 → 7.5.0

package/include/numkong/reduce/sve.h

@@ -0,0 +1,52 @@
+/**
+ *  @brief SVE horizontal reduction helpers with MSan unpoisoning.
+ *  @file include/numkong/reduce/sve.h
+ *  @author Ash Vardanian
+ *  @date April 12, 2026
+ *
+ *  LLVM's MSan does not instrument ARM SVE intrinsics — `svaddv` moves data
+ *  from vector to scalar registers via architecture-specific paths invisible
+ *  to the compiler, causing false-positive uninitialized-value reports.
+ *  These macros wrap the reduction and unpoison the scalar result.
+ *
+ *  The `svaddv` intrinsic stays inside a macro so it expands in the caller's
+ *  target context — SVE and SME streaming translation units carry incompatible
+ *  target attributes. The unpoisoning runs on the already-reduced scalar, so it
+ *  lives in a target-agnostic `NK_INTERNAL` helper called from the macro.
+ *
+ *  @sa include/numkong/reduce.h
+ */
+#ifndef NK_REDUCE_SVE_H
+#define NK_REDUCE_SVE_H
+
+#if NK_TARGET_ARM64_
+#if NK_TARGET_SVE || NK_TARGET_SVE2 || NK_TARGET_SME
+
+#include "numkong/types.h"
+
+NK_INTERNAL nk_f64_t nk_unpoison_f64_(nk_f64_t v) NK_STREAMING_COMPATIBLE_ {
+    nk_unpoison_(&v, sizeof(v));
+    return v;
+}
+NK_INTERNAL nk_f32_t nk_unpoison_f32_(nk_f32_t v) NK_STREAMING_COMPATIBLE_ {
+    nk_unpoison_(&v, sizeof(v));
+    return v;
+}
+NK_INTERNAL nk_u64_t nk_unpoison_u64_(nk_u64_t v) NK_STREAMING_COMPATIBLE_ {
+    nk_unpoison_(&v, sizeof(v));
+    return v;
+}
+NK_INTERNAL nk_i64_t nk_unpoison_i64_(nk_i64_t v) NK_STREAMING_COMPATIBLE_ {
+    nk_unpoison_(&v, sizeof(v));
+    return v;
+}
+
+#define nk_svaddv_f64_(predicate, vector) nk_unpoison_f64_(svaddv_f64((predicate), (vector)))
+#define nk_svaddv_f32_(predicate, vector) nk_unpoison_f32_(svaddv_f32((predicate), (vector)))
+#define nk_svaddv_u32_(predicate, vector) nk_unpoison_u64_(svaddv_u32((predicate), (vector)))
+#define nk_svaddv_s32_(predicate, vector) nk_unpoison_i64_(svaddv_s32((predicate), (vector)))
+#define nk_svaddv_u8_(predicate, vector)  nk_unpoison_u64_(svaddv_u8((predicate), (vector)))
+
+#endif // NK_TARGET_SVE || NK_TARGET_SVE2 || NK_TARGET_SME
+#endif // NK_TARGET_ARM64_
+#endif // NK_REDUCE_SVE_H

package/include/numkong/reduce.h

@@ -389,6 +389,8 @@ NK_PUBLIC void nk_reduce_moments_i16_neon(nk_i16_t const *, nk_size_t, nk_size_t
 /** @copydoc nk_reduce_moments_f64 */
 NK_PUBLIC void nk_reduce_moments_u16_neon(nk_u16_t const *, nk_size_t, nk_size_t, nk_u64_t *, nk_u64_t *);
 /** @copydoc nk_reduce_moments_f64 */
+NK_PUBLIC void nk_reduce_moments_u1_neon(nk_u1x8_t const *, nk_size_t, nk_size_t, nk_u64_t *, nk_u64_t *);
+/** @copydoc nk_reduce_moments_f64 */
 NK_PUBLIC void nk_reduce_moments_i32_neon(nk_i32_t const *, nk_size_t, nk_size_t, nk_i64_t *, nk_u64_t *);
 /** @copydoc nk_reduce_moments_f64 */
 NK_PUBLIC void nk_reduce_moments_u32_neon(nk_u32_t const *, nk_size_t, nk_size_t, nk_u64_t *, nk_u64_t *);
@@ -1559,6 +1561,8 @@ NK_PUBLIC void nk_reduce_moments_u1(nk_u1x8_t const *d, nk_size_t n, nk_size_t s
     nk_reduce_moments_u1_skylake(d, n, s, sum, sumsq);
 #elif NK_TARGET_HASWELL
     nk_reduce_moments_u1_haswell(d, n, s, sum, sumsq);
+#elif NK_TARGET_NEON
+    nk_reduce_moments_u1_neon(d, n, s, sum, sumsq);
 #else
     nk_reduce_moments_u1_serial(d, n, s, sum, sumsq);
 #endif

package/include/numkong/set/sve.h

@@ -32,8 +32,9 @@
 #if NK_TARGET_ARM64_
 #if NK_TARGET_SVE
 
-#include "numkong/types.h"    // `nk_u1x8_t`
-#include "numkong/set/neon.h" // `nk_hamming_u1_neon`
+#include "numkong/types.h"      // `nk_u1x8_t`
+#include "numkong/reduce/sve.h" // `nk_svaddv_f64_`
+#include "numkong/set/neon.h"   // `nk_hamming_u1_neon`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -73,7 +74,7 @@ NK_PUBLIC void nk_hamming_u1_sve(nk_u1x8_t const *a, nk_u1x8_t const *b, nk_size
             i += words_per_register;
             ++cycle;
         } while (i < n_bytes && cycle < 31);
-        differences += svaddv_u8(all_predicate_b8x, popcount_u8x);
+        differences += nk_svaddv_u8_(all_predicate_b8x, popcount_u8x);
         popcount_u8x = svdup_n_u8(0);
         cycle = 0; // Reset the cycle counter.
     }
@@ -110,9 +111,9 @@ NK_PUBLIC void nk_jaccard_u1_sve(nk_u1x8_t const *a, nk_u1x8_t const *b, nk_size
             i += words_per_register;
             ++cycle;
         } while (i < n_bytes && cycle < 31);
-        intersection_count += svaddv_u8(all_predicate_b8x, intersection_popcount_u8x);
+        intersection_count += nk_svaddv_u8_(all_predicate_b8x, intersection_popcount_u8x);
         intersection_popcount_u8x = svdup_n_u8(0);
-        union_count += svaddv_u8(all_predicate_b8x, union_popcount_u8x);
+        union_count += nk_svaddv_u8_(all_predicate_b8x, union_popcount_u8x);
         union_popcount_u8x = svdup_n_u8(0);
         cycle = 0; // Reset the cycle counter.
     }

package/include/numkong/sets/smebi32.h

@@ -41,8 +41,9 @@
 #include "numkong/types.h"
 #include "numkong/set/serial.h"
 #include "numkong/sets/serial.h"
-#include "numkong/dots/sme.h" // `nk_sme_zero_za32_*` constants
-#include "numkong/reduce.h"   // `nk_reduce_moments_u1`
+#include "numkong/reduce/sve.h"  // `nk_svaddv_f64_`
+#include "numkong/reduce/neon.h" // `nk_reduce_moments_u1_neon`
+#include "numkong/dots/sme.h"    // `nk_sme_zero_za32_*`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -100,7 +101,7 @@ NK_PUBLIC nk_u32_t nk_sets_reduce_sumsq_u1_streaming_(nk_u1x8_t const *data, nk_
         svbool_t predicate_b8x = svwhilelt_b8_u64(offset, n_bytes);
         acc_u32x = svdot_u32(acc_u32x, svcnt_u8_z(predicate_b8x, svld1_u8(predicate_b8x, data + offset)), ones_u8x);
     }
-    return (nk_u32_t)svaddv_u32(svptrue_b32(), acc_u32x);
+    return (nk_u32_t)nk_svaddv_u32_(svptrue_b32(), acc_u32x);
 }
 
 #pragma region Hamming Distance
@@ -187,11 +188,9 @@ NK_PUBLIC void nk_dots_pack_u1_smebi32(nk_u1x8_t const *b, nk_size_t row_count,
     // Compute per-row population counts
     for (nk_size_t row = 0; row < row_count; row++) {
         nk_u1x8_t const *src_row = (nk_u1x8_t const *)((char const *)b + row * b_stride_in_bytes);
-        {
-            nk_u64_t nk_local_sum_, nk_local_sumsq_;
-            nk_reduce_moments_u1(src_row, depth_bytes * 8, sizeof(nk_u1x8_t), &nk_local_sum_, &nk_local_sumsq_);
-            norms_ptr[row] = (nk_u32_t)nk_local_sum_;
-        }
+        nk_u64_t nk_local_sum_, nk_local_sumsq_;
+        nk_reduce_moments_u1_neon(src_row, depth_bytes * 8, sizeof(nk_u1x8_t), &nk_local_sum_, &nk_local_sumsq_);
+        norms_ptr[row] = (nk_u32_t)nk_local_sum_;
     }
 }
 
@@ -203,9 +202,9 @@ NK_PUBLIC void nk_dots_pack_u1_smebi32(nk_u1x8_t const *b, nk_size_t row_count,
  *  Each ZA0.S batch covers 16 depth u32 steps (one full depth tile).
  *  BMOPA expansion=1 for u32: each u32 contributes 32 bits via XNOR+POPCNT.
  */
-__arm_locally_streaming __arm_new("za") static void nk_hammings_packed_u1_smebi32_streaming_(
+__arm_new("za") static void nk_hammings_packed_u1_smebi32_streaming_( //
     nk_u1x8_t const *a, void const *b_packed, nk_u32_t *c, nk_size_t row_count_a, nk_size_t row_count_b,
-    nk_size_t depth_bits, nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+    nk_size_t depth_bits, nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) NK_STREAMING_ {
 
     nk_sets_smebi32_packed_header_t const *header = (nk_sets_smebi32_packed_header_t const *)b_packed;
     nk_size_t const row_tile_count_b = header->row_tile_count;
@@ -344,11 +343,13 @@ __arm_locally_streaming __arm_new("za") static void nk_hammings_packed_u1_smebi3
     }
 }
 
-NK_PUBLIC void nk_hammings_packed_u1_smebi32(nk_u1x8_t const *a, void const *b_packed, nk_u32_t *c,
-                                             nk_size_t row_count_a, nk_size_t row_count_b, nk_size_t depth_bits,
-                                             nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+NK_PUBLIC void nk_hammings_packed_u1_smebi32( //
+    nk_u1x8_t const *a, void const *b_packed, nk_u32_t *c, nk_size_t row_count_a, nk_size_t row_count_b,
+    nk_size_t depth_bits, nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+    nk_sme_start_streaming_();
     nk_hammings_packed_u1_smebi32_streaming_(a, b_packed, c, row_count_a, row_count_b, depth_bits, a_stride_in_bytes,
                                              c_stride_in_bytes);
+    nk_sme_stop_streaming_();
 }
 
 /**
@@ -357,9 +358,9 @@ NK_PUBLIC void nk_hammings_packed_u1_smebi32(nk_u1x8_t const *a, void const *b_p
  *  ZA1-3.S = BMOPA accumulators (3 B column tiles in fast path).
  *  Mirrors the unpacked kernel nk_hammings_packed_u1_smebi32_streaming_ pattern.
  */
-__arm_locally_streaming __arm_new("za") static void nk_hammings_symmetric_u1_smebi32_streaming_(
+__arm_new("za") static void nk_hammings_symmetric_u1_smebi32_streaming_( //
     nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t depth_bits, nk_size_t stride_in_bytes,
-    nk_u32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+    nk_u32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) NK_STREAMING_ {
 
     nk_size_t const tile_dim = svcntw();        // 16 for 512-bit SVL
     nk_size_t const depth_tile_size = svcntw(); // 16 u32 per depth tile
@@ -545,12 +546,13 @@ __arm_locally_streaming __arm_new("za") static void nk_hammings_symmetric_u1_sme
     }
 }
 
-NK_PUBLIC void nk_hammings_symmetric_u1_smebi32(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t depth_bits,
-                                                nk_size_t stride_in_bytes, nk_u32_t *result,
-                                                nk_size_t result_stride_in_bytes, nk_size_t row_start,
-                                                nk_size_t row_count) {
+NK_PUBLIC void nk_hammings_symmetric_u1_smebi32( //
+    nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t depth_bits, nk_size_t stride_in_bytes,
+    nk_u32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+    nk_sme_start_streaming_();
     nk_hammings_symmetric_u1_smebi32_streaming_(vectors, vectors_count, depth_bits, stride_in_bytes, result,
                                                 result_stride_in_bytes, row_start, row_count);
+    nk_sme_stop_streaming_();
 }
 
 #pragma endregion Hamming Distance
@@ -581,9 +583,9 @@ NK_PUBLIC void nk_hammings_symmetric_u1_smebi32(nk_u1x8_t const *vectors, nk_siz
  *    union         = (norm_a + norm_b + hamming) / 2
  *    jaccard       = 1 - intersection / union      (1.0 when union == 0)
  */
-__arm_locally_streaming __arm_new("za") static void nk_jaccards_packed_u1_smebi32_streaming_(
+__arm_new("za") static void nk_jaccards_packed_u1_smebi32_streaming_( //
     nk_u1x8_t const *a, void const *b_packed, nk_f32_t *c, nk_size_t row_count_a, nk_size_t row_count_b,
-    nk_size_t depth_bits, nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+    nk_size_t depth_bits, nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) NK_STREAMING_ {
 
     nk_sets_smebi32_packed_header_t const *header = (nk_sets_smebi32_packed_header_t const *)b_packed;
     nk_size_t const row_tile_count_b = header->row_tile_count;
@@ -796,11 +798,13 @@ __arm_locally_streaming __arm_new("za") static void nk_jaccards_packed_u1_smebi3
     }
 }
 
-NK_PUBLIC void nk_jaccards_packed_u1_smebi32(nk_u1x8_t const *a, void const *b_packed, nk_f32_t *c,
-                                             nk_size_t row_count_a, nk_size_t row_count_b, nk_size_t depth_bits,
-                                             nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+NK_PUBLIC void nk_jaccards_packed_u1_smebi32( //
+    nk_u1x8_t const *a, void const *b_packed, nk_f32_t *c, nk_size_t row_count_a, nk_size_t row_count_b,
+    nk_size_t depth_bits, nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+    nk_sme_start_streaming_();
     nk_jaccards_packed_u1_smebi32_streaming_(a, b_packed, c, row_count_a, row_count_b, depth_bits, a_stride_in_bytes,
                                              c_stride_in_bytes);
+    nk_sme_stop_streaming_();
 }
 
 /**
@@ -808,9 +812,9 @@ NK_PUBLIC void nk_jaccards_packed_u1_smebi32(nk_u1x8_t const *a, void const *b_p
  *  Fills upper triangle only (column_tile >= row_tile); caller sees result[i][j] for j >= i.
  *  Norms computed on-the-fly using streaming SVE popcount.
  */
-__arm_locally_streaming __arm_new("za") static void nk_jaccards_symmetric_u1_smebi32_streaming_(
+__arm_new("za") static void nk_jaccards_symmetric_u1_smebi32_streaming_( //
     nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t depth_bits, nk_size_t stride_in_bytes,
-    nk_f32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+    nk_f32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) NK_STREAMING_ {
 
     nk_size_t const tile_dim = svcntw();        // 16 for 512-bit SVL
     nk_size_t const depth_tile_size = svcntw(); // 16 u32 per depth tile
@@ -1104,12 +1108,13 @@ __arm_locally_streaming __arm_new("za") static void nk_jaccards_symmetric_u1_sme
     }
 }
 
-NK_PUBLIC void nk_jaccards_symmetric_u1_smebi32(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t depth_bits,
-                                                nk_size_t stride_in_bytes, nk_f32_t *result,
-                                                nk_size_t result_stride_in_bytes, nk_size_t row_start,
-                                                nk_size_t row_count) {
+NK_PUBLIC void nk_jaccards_symmetric_u1_smebi32( //
+    nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t depth_bits, nk_size_t stride_in_bytes,
+    nk_f32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+    nk_sme_start_streaming_();
     nk_jaccards_symmetric_u1_smebi32_streaming_(vectors, vectors_count, depth_bits, stride_in_bytes, result,
                                                 result_stride_in_bytes, row_start, row_count);
+    nk_sme_stop_streaming_();
 }
 
 #pragma endregion Jaccard Distance

package/include/numkong/sparse/sve2.h

@@ -12,6 +12,7 @@
 #if NK_TARGET_ARM64_
 
 #include "numkong/types.h"
+#include "numkong/reduce/sve.h" // `nk_svaddv_f64_`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -395,7 +396,7 @@ NK_PUBLIC void nk_sparse_dot_u32f32_sve2(                 //
         a_idx += a_step;
         b_idx += b_step;
     }
-    *product = svaddv_f64(predicate_all_b64x, product_f64x);
+    *product = nk_svaddv_f64_(predicate_all_b64x, product_f64x);
 }
 
 #if defined(__clang__)
@@ -485,7 +486,7 @@ NK_PUBLIC void nk_sparse_dot_u16bf16_sve2(                  //
         a_idx += a_step;
         b_idx += b_step;
     }
-    *product = svaddv_f32(svptrue_b32(), product_f32x);
+    *product = nk_svaddv_f32_(svptrue_b32(), product_f32x);
 }
 
 #if defined(__clang__)

package/include/numkong/spatial/sve.h

@@ -36,6 +36,7 @@
 #if NK_TARGET_SVE
 
 #include "numkong/types.h"
+#include "numkong/reduce/sve.h"   // `nk_svaddv_f64_`
 #include "numkong/spatial/neon.h" // `nk_f64_sqrt_neon`
 #include "numkong/dot/sve.h"      // `nk_dot_stable_sum_f64_sve_`
 
@@ -113,7 +114,7 @@ NK_PUBLIC void nk_sqeuclidean_f32_sve(nk_f32_t const *a, nk_f32_t const *b, nk_s
         svfloat64_t diff_odd_f64x = svsub_f64_x(pred_odd_b64x, a_odd_f64x, b_odd_f64x);
         dist_sq_f64x = svmla_f64_m(pred_odd_b64x, dist_sq_f64x, diff_odd_f64x, diff_odd_f64x);
     }
-    nk_f64_t dist_sq_f64 = svaddv_f64(svptrue_b64(), dist_sq_f64x);
+    nk_f64_t dist_sq_f64 = nk_svaddv_f64_(svptrue_b64(), dist_sq_f64x);
     *result = dist_sq_f64;
 }
 
@@ -149,9 +150,9 @@ NK_PUBLIC void nk_angular_f32_sve(nk_f32_t const *a, nk_f32_t const *b, nk_size_
         b2_f64x = svmla_f64_m(pred_odd_b64x, b2_f64x, b_odd_f64x, b_odd_f64x);
     }
 
-    nk_f64_t ab_f64 = svaddv_f64(svptrue_b64(), ab_f64x);
-    nk_f64_t a2_f64 = svaddv_f64(svptrue_b64(), a2_f64x);
-    nk_f64_t b2_f64 = svaddv_f64(svptrue_b64(), b2_f64x);
+    nk_f64_t ab_f64 = nk_svaddv_f64_(svptrue_b64(), ab_f64x);
+    nk_f64_t a2_f64 = nk_svaddv_f64_(svptrue_b64(), a2_f64x);
+    nk_f64_t b2_f64 = nk_svaddv_f64_(svptrue_b64(), b2_f64x);
     *result = nk_angular_normalize_f64_neon_(ab_f64, a2_f64, b2_f64);
 }
 
@@ -225,8 +226,8 @@ NK_PUBLIC void nk_angular_f64_sve(nk_f64_t const *a, nk_f64_t const *b, nk_size_
     } while (i < n);
 
     nk_f64_t ab_f64 = nk_dot_stable_sum_f64_sve_(predicate_all_b64x, ab_sum_f64x, ab_compensation_f64x);
-    nk_f64_t a2_f64 = svaddv_f64(predicate_all_b64x, a2_f64x);
-    nk_f64_t b2_f64 = svaddv_f64(predicate_all_b64x, b2_f64x);
+    nk_f64_t a2_f64 = nk_svaddv_f64_(predicate_all_b64x, a2_f64x);
+    nk_f64_t b2_f64 = nk_svaddv_f64_(predicate_all_b64x, b2_f64x);
     *result = nk_angular_normalize_f64_neon_(ab_f64, a2_f64, b2_f64);
 }
 

package/include/numkong/spatial/svebfdot.h

@@ -36,6 +36,7 @@
 #if NK_TARGET_SVEBFDOT
 
 #include "numkong/types.h"
+#include "numkong/reduce/sve.h"   // `nk_svaddv_f64_`
 #include "numkong/spatial/neon.h" // `nk_f32_sqrt_neon`
 
 #if defined(__cplusplus)
@@ -75,7 +76,9 @@ NK_PUBLIC void nk_sqeuclidean_bf16_svebfdot(nk_bf16_t const *a_enum, nk_bf16_t c
         d2_high_f32x = svmla_f32_m(predicate_high_b32x, d2_high_f32x, a_minus_b_high_f32x, a_minus_b_high_f32x);
         i += svcnth();
     } while (i < n);
-    nk_f32_t d2 = svaddv_f32(svptrue_b32(), d2_low_f32x) + svaddv_f32(svptrue_b32(), d2_high_f32x);
+    nk_f32_t d2_low = nk_svaddv_f32_(svptrue_b32(), d2_low_f32x);
+    nk_f32_t d2_high = nk_svaddv_f32_(svptrue_b32(), d2_high_f32x);
+    nk_f32_t d2 = d2_low + d2_high;
     *result = d2;
 }
 NK_PUBLIC void nk_euclidean_bf16_svebfdot(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t *result) {
@@ -101,9 +104,9 @@ NK_PUBLIC void nk_angular_bf16_svebfdot(nk_bf16_t const *a_enum, nk_bf16_t const
         i += svcnth();
     } while (i < n);
 
-    nk_f32_t ab = svaddv_f32(svptrue_b32(), ab_f32x);
-    nk_f32_t a2 = svaddv_f32(svptrue_b32(), a2_f32x);
-    nk_f32_t b2 = svaddv_f32(svptrue_b32(), b2_f32x);
+    nk_f32_t ab = nk_svaddv_f32_(svptrue_b32(), ab_f32x);
+    nk_f32_t a2 = nk_svaddv_f32_(svptrue_b32(), a2_f32x);
+    nk_f32_t b2 = nk_svaddv_f32_(svptrue_b32(), b2_f32x);
     *result = nk_angular_normalize_f32_neon_(ab, a2, b2);
 }
 

package/include/numkong/spatial/svehalf.h

@@ -32,6 +32,7 @@
 #if NK_TARGET_SVEHALF
 
 #include "numkong/types.h"
+#include "numkong/reduce/sve.h"   // `nk_svaddv_f64_`
 #include "numkong/spatial/neon.h" // `nk_f32_sqrt_neon`
 
 #if defined(__cplusplus)
@@ -74,7 +75,7 @@ NK_PUBLIC void nk_sqeuclidean_f16_svehalf(nk_f16_t const *a_enum, nk_f16_t const
 
         i += svcnth();
     } while (i < n);
-    *result = svaddv_f32(svptrue_b32(), d2_f32x);
+    *result = nk_svaddv_f32_(svptrue_b32(), d2_f32x);
 }
 
 NK_PUBLIC void nk_euclidean_f16_svehalf(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *result) {
@@ -114,9 +115,9 @@ NK_PUBLIC void nk_angular_f16_svehalf(nk_f16_t const *a_enum, nk_f16_t const *b_
         i += svcnth();
     } while (i < n);
 
-    nk_f32_t ab_f32 = svaddv_f32(svptrue_b32(), ab_f32x);
-    nk_f32_t a2_f32 = svaddv_f32(svptrue_b32(), a2_f32x);
-    nk_f32_t b2_f32 = svaddv_f32(svptrue_b32(), b2_f32x);
+    nk_f32_t ab_f32 = nk_svaddv_f32_(svptrue_b32(), ab_f32x);
+    nk_f32_t a2_f32 = nk_svaddv_f32_(svptrue_b32(), a2_f32x);
+    nk_f32_t b2_f32 = nk_svaddv_f32_(svptrue_b32(), b2_f32x);
     *result = nk_angular_normalize_f32_neon_(ab_f32, a2_f32, b2_f32);
 }
 

package/include/numkong/spatial/svesdot.h

@@ -34,6 +34,7 @@
 #if NK_TARGET_SVESDOT
 
 #include "numkong/types.h"
+#include "numkong/reduce/sve.h"   // `nk_svaddv_f64_`
 #include "numkong/spatial/neon.h" // `nk_angular_normalize_f32_neon_`, `nk_f32_sqrt_neon`
 
 #if defined(__cplusplus)
@@ -58,7 +59,7 @@ NK_PUBLIC void nk_sqeuclidean_i8_svesdot(nk_i8_t const *a, nk_i8_t const *b, nk_
         distance_sq_u32x = svdot_u32(distance_sq_u32x, diff_u8x, diff_u8x);
         i += svcntb();
     } while (i < n);
-    *result = (nk_u32_t)svaddv_u32(svptrue_b32(), distance_sq_u32x);
+    *result = (nk_u32_t)nk_svaddv_u32_(svptrue_b32(), distance_sq_u32x);
 }
 NK_PUBLIC void nk_euclidean_i8_svesdot(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_f32_t *result) {
     nk_u32_t distance_sq_u32;
@@ -81,9 +82,9 @@ NK_PUBLIC void nk_angular_i8_svesdot(nk_i8_t const *a, nk_i8_t const *b, nk_size
         i += svcntb();
     } while (i < n);
 
-    nk_i32_t ab = (nk_i32_t)svaddv_s32(svptrue_b32(), ab_i32x);
-    nk_i32_t a2 = (nk_i32_t)svaddv_s32(svptrue_b32(), a2_i32x);
-    nk_i32_t b2 = (nk_i32_t)svaddv_s32(svptrue_b32(), b2_i32x);
+    nk_i32_t ab = (nk_i32_t)nk_svaddv_s32_(svptrue_b32(), ab_i32x);
+    nk_i32_t a2 = (nk_i32_t)nk_svaddv_s32_(svptrue_b32(), a2_i32x);
+    nk_i32_t b2 = (nk_i32_t)nk_svaddv_s32_(svptrue_b32(), b2_i32x);
     *result = nk_angular_normalize_f32_neon_((nk_f32_t)ab, (nk_f32_t)a2, (nk_f32_t)b2);
 }
 
@@ -98,7 +99,7 @@ NK_PUBLIC void nk_sqeuclidean_u8_svesdot(nk_u8_t const *a, nk_u8_t const *b, nk_
         distance_sq_u32x = svdot_u32(distance_sq_u32x, diff_u8x, diff_u8x);
         i += svcntb();
     } while (i < n);
-    *result = (nk_u32_t)svaddv_u32(svptrue_b32(), distance_sq_u32x);
+    *result = (nk_u32_t)nk_svaddv_u32_(svptrue_b32(), distance_sq_u32x);
 }
 NK_PUBLIC void nk_euclidean_u8_svesdot(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_f32_t *result) {
     nk_u32_t distance_sq_u32;
@@ -121,9 +122,9 @@ NK_PUBLIC void nk_angular_u8_svesdot(nk_u8_t const *a, nk_u8_t const *b, nk_size
         i += svcntb();
     } while (i < n);
 
-    nk_u32_t ab = (nk_u32_t)svaddv_u32(svptrue_b32(), ab_u32x);
-    nk_u32_t a2 = (nk_u32_t)svaddv_u32(svptrue_b32(), a2_u32x);
-    nk_u32_t b2 = (nk_u32_t)svaddv_u32(svptrue_b32(), b2_u32x);
+    nk_u32_t ab = (nk_u32_t)nk_svaddv_u32_(svptrue_b32(), ab_u32x);
+    nk_u32_t a2 = (nk_u32_t)nk_svaddv_u32_(svptrue_b32(), a2_u32x);
+    nk_u32_t b2 = (nk_u32_t)nk_svaddv_u32_(svptrue_b32(), b2_u32x);
     *result = nk_angular_normalize_f32_neon_((nk_f32_t)ab, (nk_f32_t)a2, (nk_f32_t)b2);
 }
 

package/include/numkong/spatials/graniteamx.h

@@ -0,0 +1,173 @@
+/**
+ *  @brief Batched Spatial Distances for Granite Rapids (AMX-FP16) with AVX-512 Finalization.
+ *  @file include/numkong/spatials/graniteamx.h
+ *  @author Ash Vardanian
+ *  @date April 9, 2026
+ *
+ *  @sa include/numkong/spatials.h
+ */
+#ifndef NK_SPATIALS_GRANITEAMX_H
+#define NK_SPATIALS_GRANITEAMX_H
+
+#if NK_TARGET_X8664_
+#if NK_TARGET_GRANITEAMX
+
+#include "numkong/spatial/skylake.h"
+#include "numkong/spatial/serial.h"
+#include "numkong/dots/graniteamx.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(                                                                                                     \
+    __attribute__((target(                                                                                                        \
+        "avx2,avx512f,avx512vl,avx512bw,avx512dq,avx512fp16,avx512vbmi,f16c,fma,bmi,bmi2,amx-tile,amx-bf16,amx-int8,amx-fp16"))), \
+    apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "avx512dq", "avx512fp16", "avx512vbmi", "f16c", "fma", \
+                   "bmi", "bmi2", "amx-tile", "amx-bf16", "amx-int8", "amx-fp16")
+#endif
+
+#pragma region F16 Packed
+
+NK_INTERNAL void nk_angulars_packed_f16_graniteamx_finalize_(nk_f16_t const *a, void const *b_packed, nk_f32_t *c,
+                                                             nk_size_t rows, nk_size_t columns, nk_size_t depth,
+                                                             nk_size_t a_stride_elements, nk_size_t c_stride_elements) {
+    nk_dots_amx_packed_header_t const *header = (nk_dots_amx_packed_header_t const *)b_packed;
+    nk_f32_t const *b_norms = (nk_f32_t const *)((char const *)b_packed + header->norms_byte_offset);
+    for (nk_size_t row = 0; row < rows; row++) {
+        nk_f32_t query_norm_sq = nk_dots_reduce_sumsq_f16_(a + row * a_stride_elements, depth);
+        nk_angulars_row_f32dots_sapphireamx_(c + row * c_stride_elements, b_norms, query_norm_sq, columns);
+    }
+}
+
+NK_PUBLIC void nk_angulars_packed_f16_graniteamx(         //
+    nk_f16_t const *a, void const *b_packed, nk_f32_t *c, //
+    nk_size_t rows, nk_size_t columns, nk_size_t depth,   //
+    nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+    nk_size_t const a_stride_elements = a_stride_in_bytes / sizeof(nk_f16_t);
+    nk_size_t const c_stride_elements = c_stride_in_bytes / sizeof(nk_f32_t);
+    nk_dots_packed_f16_graniteamx(a, b_packed, c, rows, columns, depth, a_stride_in_bytes, c_stride_in_bytes);
+    nk_angulars_packed_f16_graniteamx_finalize_(a, b_packed, c, rows, columns, depth, a_stride_elements,
+                                                c_stride_elements);
+}
+
+NK_INTERNAL void nk_euclideans_packed_f16_graniteamx_finalize_(nk_f16_t const *a, void const *b_packed, nk_f32_t *c,
+                                                               nk_size_t rows, nk_size_t columns, nk_size_t depth,
+                                                               nk_size_t a_stride_elements,
+                                                               nk_size_t c_stride_elements) {
+    nk_dots_amx_packed_header_t const *header = (nk_dots_amx_packed_header_t const *)b_packed;
+    nk_f32_t const *b_norms = (nk_f32_t const *)((char const *)b_packed + header->norms_byte_offset);
+    for (nk_size_t row = 0; row < rows; row++) {
+        nk_f32_t query_norm_sq = nk_dots_reduce_sumsq_f16_(a + row * a_stride_elements, depth);
+        nk_euclideans_row_f32dots_sapphireamx_(c + row * c_stride_elements, b_norms, query_norm_sq, columns);
+    }
+}
+
+NK_PUBLIC void nk_euclideans_packed_f16_graniteamx(       //
+    nk_f16_t const *a, void const *b_packed, nk_f32_t *c, //
+    nk_size_t rows, nk_size_t columns, nk_size_t depth,   //
+    nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+    nk_size_t const a_stride_elements = a_stride_in_bytes / sizeof(nk_f16_t);
+    nk_size_t const c_stride_elements = c_stride_in_bytes / sizeof(nk_f32_t);
+    nk_dots_packed_f16_graniteamx(a, b_packed, c, rows, columns, depth, a_stride_in_bytes, c_stride_in_bytes);
+    nk_euclideans_packed_f16_graniteamx_finalize_(a, b_packed, c, rows, columns, depth, a_stride_elements,
+                                                  c_stride_elements);
+}
+
+#pragma endregion F16 Packed
+
+#pragma region F16 Symmetric
+
+NK_INTERNAL void nk_angulars_symmetric_f16_graniteamx_finalize_(nk_f16_t const *vectors, nk_size_t vectors_count,
+                                                                nk_size_t depth, nk_size_t stride_elements,
+                                                                nk_f32_t *result, nk_size_t result_stride_elements,
+                                                                nk_size_t row_start, nk_size_t row_count) {
+
+    for (nk_size_t row = row_start; row < row_start + row_count; row++)
+        result[row * result_stride_elements + row] = nk_dots_reduce_sumsq_f16_(vectors + row * stride_elements, depth);
+
+    nk_f32_t column_norms_cache[256];
+    for (nk_size_t chunk_start = 0; chunk_start < vectors_count; chunk_start += 256) {
+        nk_size_t chunk_end = chunk_start + 256 < vectors_count ? chunk_start + 256 : vectors_count;
+        for (nk_size_t col = chunk_start; col < chunk_end; col++)
+            column_norms_cache[col - chunk_start] = nk_dots_reduce_sumsq_f16_(vectors + col * stride_elements, depth);
+
+        for (nk_size_t row = row_start; row < row_start + row_count; row++) {
+            nk_f32_t *r_row = result + row * result_stride_elements;
+            nk_size_t col_start = chunk_start > row + 1 ? chunk_start : row + 1;
+            if (col_start >= chunk_end) continue;
+            nk_angulars_row_f32dots_sapphireamx_(r_row + col_start, column_norms_cache + col_start - chunk_start,
+                                                 r_row[row], chunk_end - col_start);
+        }
+    }
+
+    for (nk_size_t row = row_start; row < row_start + row_count; row++) result[row * result_stride_elements + row] = 0;
+}
+
+NK_PUBLIC void nk_angulars_symmetric_f16_graniteamx(                                              //
+    nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_in_bytes, //
+    nk_f32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+    nk_size_t const stride_elements = stride_in_bytes / sizeof(nk_f16_t);
+    nk_size_t const result_stride_elements = result_stride_in_bytes / sizeof(nk_f32_t);
+    nk_dots_symmetric_f16_graniteamx(vectors, vectors_count, depth, stride_in_bytes, result, result_stride_in_bytes,
+                                     row_start, row_count);
+    nk_angulars_symmetric_f16_graniteamx_finalize_(vectors, vectors_count, depth, stride_elements, result,
+                                                   result_stride_elements, row_start, row_count);
+}
+
+NK_INTERNAL void nk_euclideans_symmetric_f16_graniteamx_finalize_(nk_f16_t const *vectors, nk_size_t vectors_count,
+                                                                  nk_size_t depth, nk_size_t stride_elements,
+                                                                  nk_f32_t *result, nk_size_t result_stride_elements,
+                                                                  nk_size_t row_start, nk_size_t row_count) {
+
+    for (nk_size_t row = row_start; row < row_start + row_count; row++)
+        result[row * result_stride_elements + row] = nk_dots_reduce_sumsq_f16_(vectors + row * stride_elements, depth);
+
+    nk_f32_t column_norms_cache[256];
+    for (nk_size_t chunk_start = 0; chunk_start < vectors_count; chunk_start += 256) {
+        nk_size_t chunk_end = chunk_start + 256 < vectors_count ? chunk_start + 256 : vectors_count;
+        for (nk_size_t col = chunk_start; col < chunk_end; col++)
+            column_norms_cache[col - chunk_start] = nk_dots_reduce_sumsq_f16_(vectors + col * stride_elements, depth);
+
+        for (nk_size_t row = row_start; row < row_start + row_count; row++) {
+            nk_f32_t *r_row = result + row * result_stride_elements;
+            nk_size_t col_start = chunk_start > row + 1 ? chunk_start : row + 1;
+            if (col_start >= chunk_end) continue;
+            nk_euclideans_row_f32dots_sapphireamx_(r_row + col_start, column_norms_cache + col_start - chunk_start,
+                                                   r_row[row], chunk_end - col_start);
+        }
+    }
+
+    for (nk_size_t row = row_start; row < row_start + row_count; row++) result[row * result_stride_elements + row] = 0;
+}
+
+NK_PUBLIC void nk_euclideans_symmetric_f16_graniteamx(                                            //
+    nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_in_bytes, //
+    nk_f32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+    nk_size_t const stride_elements = stride_in_bytes / sizeof(nk_f16_t);
+    nk_size_t const result_stride_elements = result_stride_in_bytes / sizeof(nk_f32_t);
+    nk_dots_symmetric_f16_graniteamx(vectors, vectors_count, depth, stride_in_bytes, result, result_stride_in_bytes,
+                                     row_start, row_count);
+    nk_euclideans_symmetric_f16_graniteamx_finalize_(vectors, vectors_count, depth, stride_elements, result,
+                                                     result_stride_elements, row_start, row_count);
+}
+
+#pragma endregion F16 Symmetric
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_GRANITEAMX
+#endif // NK_TARGET_X8664_
+#endif // NK_SPATIALS_GRANITEAMX_H

package/include/numkong/spatials/serial.h

@@ -15,6 +15,18 @@
 extern "C" {
 #endif
 
+/* Optimize serial fallbacks for size — see dots/serial.h for rationale. */
+#if defined(NDEBUG)
+#if defined(_MSC_VER)
+#pragma optimize("s", on)
+#elif defined(__clang__)
+#pragma clang attribute push(__attribute__((minsize)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("Os")
+#endif
+#endif
+
 nk_define_cross_normalized_packed_(angular, f64, serial, f64, f64, f64, /*norm_value_type=*/f64, f64, nk_b256_vec_t,
                                    nk_dots_packed_f64_serial, nk_angular_through_f64_from_dot_serial_,
                                    nk_dots_reduce_sumsq_f64_, nk_load_b256_serial_, nk_partial_load_b64x4_serial_,
@@ -219,6 +231,16 @@ nk_define_cross_normalized_symmetric_(euclidean, u4, serial, u4x2, u32, /*norm_v
                                       nk_dots_reduce_sumsq_u4_, nk_load_b128_serial_, nk_partial_load_b32x4_serial_,
                                       nk_store_b128_serial_, nk_partial_store_b32x4_serial_, 2)
 
+#if defined(NDEBUG)
+#if defined(_MSC_VER)
+#pragma optimize("", on)
+#elif defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+#endif
+
 #if defined(__cplusplus)
 } // extern "C"
 #endif