numkong 7.4.5 → 7.6.0

package/include/numkong/dots/smebi32.h

@@ -39,9 +39,9 @@ extern "C" {
  *  BMOPA gives matching = popcount(XNOR(a,b)).
  *  dot(a,b) = popcount(a AND b) = (pop_a + pop_b - depth_bits + matching) / 2
  */
-__arm_locally_streaming __arm_new("za") static void nk_dots_packed_u1_smebi32_streaming_(
+__arm_new("za") static void nk_dots_packed_u1_smebi32_streaming_( //
     nk_u1x8_t const *a, void const *b_packed, nk_u32_t *c, nk_size_t row_count_a, nk_size_t row_count_b,
-    nk_size_t depth_bits, nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+    nk_size_t depth_bits, nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) NK_STREAMING_ {
 
     nk_sets_smebi32_packed_header_t const *header = (nk_sets_smebi32_packed_header_t const *)b_packed;
     nk_size_t const row_tile_count_b = header->row_tile_count;
@@ -204,20 +204,22 @@ __arm_locally_streaming __arm_new("za") static void nk_dots_packed_u1_smebi32_st
     }
 }
 
-NK_PUBLIC void nk_dots_packed_u1_smebi32(nk_u1x8_t const *a, void const *b_packed, nk_u32_t *c, nk_size_t row_count_a,
-                                         nk_size_t row_count_b, nk_size_t depth_bits, nk_size_t a_stride_in_bytes,
-                                         nk_size_t c_stride_in_bytes) {
+NK_PUBLIC void nk_dots_packed_u1_smebi32( //
+    nk_u1x8_t const *a, void const *b_packed, nk_u32_t *c, nk_size_t row_count_a, nk_size_t row_count_b,
+    nk_size_t depth_bits, nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
+    nk_sme_start_streaming_();
     nk_dots_packed_u1_smebi32_streaming_(a, b_packed, c, row_count_a, row_count_b, depth_bits, a_stride_in_bytes,
                                          c_stride_in_bytes);
+    nk_sme_stop_streaming_();
 }
 
 /**
  *  Symmetric u1 dot-product using ZA0 time-sharing + 3-tile fast path.
  *  Same ZA transpose pattern as hammings_symmetric, but with dot extraction.
  */
-__arm_locally_streaming __arm_new("za") static void nk_dots_symmetric_u1_smebi32_streaming_(
+__arm_new("za") static void nk_dots_symmetric_u1_smebi32_streaming_( //
     nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t depth_bits, nk_size_t stride_in_bytes,
-    nk_u32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+    nk_u32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) NK_STREAMING_ {
 
     nk_size_t const tile_dim = svcntw();        // 16 for 512-bit SVL
     nk_size_t const depth_tile_size = svcntw(); // 16 u32 per depth tile
@@ -451,12 +453,13 @@ __arm_locally_streaming __arm_new("za") static void nk_dots_symmetric_u1_smebi32
     }
 }
 
-NK_PUBLIC void nk_dots_symmetric_u1_smebi32(nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t depth_bits,
-                                            nk_size_t stride_in_bytes, nk_u32_t *result,
-                                            nk_size_t result_stride_in_bytes, nk_size_t row_start,
-                                            nk_size_t row_count) {
+NK_PUBLIC void nk_dots_symmetric_u1_smebi32( //
+    nk_u1x8_t const *vectors, nk_size_t vectors_count, nk_size_t depth_bits, nk_size_t stride_in_bytes,
+    nk_u32_t *result, nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
+    nk_sme_start_streaming_();
     nk_dots_symmetric_u1_smebi32_streaming_(vectors, vectors_count, depth_bits, stride_in_bytes, result,
                                             result_stride_in_bytes, row_start, row_count);
+    nk_sme_stop_streaming_();
 }
 
 #if defined(__clang__)

package/include/numkong/dots/smef64.h

@@ -153,9 +153,9 @@ NK_PUBLIC void nk_dots_pack_f32_smef64(nk_f32_t const *b, nk_size_t columns, nk_
     }
 }
 
-__arm_locally_streaming __arm_new("za") static void nk_dots_packed_f32_smef64_streaming_(
+__arm_new("za") static void nk_dots_packed_f32_smef64_streaming_( //
     nk_f32_t const *a, void const *b_packed, nk_f64_t *c, nk_size_t rows, nk_size_t columns, nk_size_t depth,
-    nk_size_t a_stride_elements, nk_size_t c_stride_elements) {
+    nk_size_t a_stride_elements, nk_size_t c_stride_elements) NK_STREAMING_ {
 
     nk_dots_sme_packed_header_t const *header = (nk_dots_sme_packed_header_t const *)b_packed;
     nk_size_t const column_tile_count = header->column_tile_count;
@@ -390,14 +390,16 @@ __arm_locally_streaming __arm_new("za") static void nk_dots_packed_f32_smef64_st
     }
 }
 
-NK_PUBLIC void nk_dots_packed_f32_smef64(nk_f32_t const *a, void const *b_packed, nk_f64_t *c, nk_size_t rows,
-                                         nk_size_t columns, nk_size_t depth, nk_size_t a_stride_in_bytes,
-                                         nk_size_t c_stride_in_bytes) {
+NK_PUBLIC void nk_dots_packed_f32_smef64( //
+    nk_f32_t const *a, void const *b_packed, nk_f64_t *c, nk_size_t rows, nk_size_t columns, nk_size_t depth,
+    nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
 
     nk_size_t const a_stride_elements = a_stride_in_bytes / sizeof(nk_f32_t);
     nk_size_t const c_stride_elements = c_stride_in_bytes / sizeof(nk_f64_t);
 
+    nk_sme_start_streaming_();
     nk_dots_packed_f32_smef64_streaming_(a, b_packed, c, rows, columns, depth, a_stride_elements, c_stride_elements);
+    nk_sme_stop_streaming_();
 }
 
 /**
@@ -406,9 +408,9 @@ NK_PUBLIC void nk_dots_packed_f32_smef64(nk_f32_t const *a, void const *b_packed
  *  pre-reads A columns into Z registers, then reloads ZA0 with widened B data
  *  per column tile. Eliminates all scalar B-packing loops.
  */
-__arm_locally_streaming __arm_new("za") static void nk_dots_symmetric_f32_smef64_streaming_(
+__arm_new("za") static void nk_dots_symmetric_f32_smef64_streaming_( //
     nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_elements, nk_f64_t *result,
-    nk_size_t result_stride_elements, nk_size_t row_start, nk_size_t row_count) {
+    nk_size_t result_stride_elements, nk_size_t row_start, nk_size_t row_count) NK_STREAMING_ {
 
     nk_size_t const tile_dimension = svcntd();              // 8 for SVL=512
     nk_size_t const depth_tile_size = svcntw();             // 16 for SVL=512
@@ -721,15 +723,16 @@ __arm_locally_streaming __arm_new("za") static void nk_dots_symmetric_f32_smef64
     }
 }
 
-NK_PUBLIC void nk_dots_symmetric_f32_smef64(nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
-                                            nk_size_t stride_in_bytes, nk_f64_t *result,
-                                            nk_size_t result_stride_in_bytes, nk_size_t row_start,
-                                            nk_size_t row_count) {
+NK_PUBLIC void nk_dots_symmetric_f32_smef64( //
+    nk_f32_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_in_bytes, nk_f64_t *result,
+    nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
 
     nk_size_t const stride_elements = stride_in_bytes / sizeof(nk_f32_t);
     nk_size_t const result_stride_elements = result_stride_in_bytes / sizeof(nk_f64_t);
+    nk_sme_start_streaming_();
     nk_dots_symmetric_f32_smef64_streaming_(vectors, vectors_count, depth, stride_elements, result,
                                             result_stride_elements, row_start, row_count);
+    nk_sme_stop_streaming_();
 }
 
 #pragma endregion F32 Floats
@@ -783,17 +786,16 @@ NK_PUBLIC void nk_dots_symmetric_f32_smef64(nk_f32_t const *vectors, nk_size_t v
  *
  *  All slices fit in f32 (24-bit significand). Products: max 19+19 = 38 ≤ 53, exact in f64.
  */
-NK_PUBLIC nk_u64_t nk_f64_smef64_ozaki_mask_19_bits_(void) NK_STREAMING_ {
+NK_PUBLIC nk_u64_t nk_f64_smef64_ozaki_mask_19_bits_(void) {
     return 0xFFFFFFFC00000000ULL; // keep top 19 sig bits
 }
-NK_PUBLIC nk_u64_t nk_f64_smef64_ozaki_mask_17_bits_(void) NK_STREAMING_ {
+NK_PUBLIC nk_u64_t nk_f64_smef64_ozaki_mask_17_bits_(void) {
     return 0xFFFFFFF000000000ULL; // keep top 17 sig bits
 }
 
 /*  Split a scalar f64 into 3 non-overlapping Ozaki slices (19+17+17 mantissa bits).
  *  Each slice fits in f32. Outputs stored via pointers. */
-NK_PUBLIC void nk_f64_smef64_ozaki_split_f64_(nk_f64_t val, nk_f64_t *slice_0, nk_f64_t *slice_1,
-                                              nk_f64_t *slice_2) NK_STREAMING_ {
+NK_PUBLIC void nk_f64_smef64_ozaki_split_f64_(nk_f64_t val, nk_f64_t *slice_0, nk_f64_t *slice_1, nk_f64_t *slice_2) {
     nk_fui64_t pun;
     pun.f = val;
     pun.u &= nk_f64_smef64_ozaki_mask_19_bits_();
@@ -805,9 +807,9 @@ NK_PUBLIC void nk_f64_smef64_ozaki_split_f64_(nk_f64_t val, nk_f64_t *slice_0, n
     *slice_2 = residual - *slice_1;
 }
 
-__arm_locally_streaming __arm_new("za") static void nk_dots_symmetric_f64_smef64_streaming_(
+__arm_new("za") static void nk_dots_symmetric_f64_smef64_streaming_( //
     nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_elements, nk_f64_t *result,
-    nk_size_t result_stride_elements, nk_size_t row_start, nk_size_t row_count) {
+    nk_size_t result_stride_elements, nk_size_t row_start, nk_size_t row_count) NK_STREAMING_ {
 
     nk_size_t const tile_dimension = svcntd();
     nk_size_t const depth_steps_per_batch = tile_dimension;
@@ -929,15 +931,16 @@ __arm_locally_streaming __arm_new("za") static void nk_dots_symmetric_f64_smef64
     }
 }
 
-NK_PUBLIC void nk_dots_symmetric_f64_smef64(nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
-                                            nk_size_t stride_in_bytes, nk_f64_t *result,
-                                            nk_size_t result_stride_in_bytes, nk_size_t row_start,
-                                            nk_size_t row_count) {
+NK_PUBLIC void nk_dots_symmetric_f64_smef64( //
+    nk_f64_t const *vectors, nk_size_t vectors_count, nk_size_t depth, nk_size_t stride_in_bytes, nk_f64_t *result,
+    nk_size_t result_stride_in_bytes, nk_size_t row_start, nk_size_t row_count) {
 
     nk_size_t const stride_elements = stride_in_bytes / sizeof(nk_f64_t);
     nk_size_t const result_stride_elements = result_stride_in_bytes / sizeof(nk_f64_t);
+    nk_sme_start_streaming_();
     nk_dots_symmetric_f64_smef64_streaming_(vectors, vectors_count, depth, stride_elements, result,
                                             result_stride_elements, row_start, row_count);
+    nk_sme_stop_streaming_();
 }
 
 NK_PUBLIC nk_size_t nk_dots_packed_size_f64_smef64(nk_size_t columns, nk_size_t depth) {
@@ -1018,9 +1021,9 @@ NK_PUBLIC void nk_dots_pack_f64_smef64(nk_f64_t const *b, nk_size_t columns, nk_
     }
 }
 
-__arm_locally_streaming __arm_new("za") static void nk_dots_packed_f64_smef64_streaming_(
+__arm_new("za") static void nk_dots_packed_f64_smef64_streaming_( //
     nk_f64_t const *a, void const *b_packed, nk_f64_t *c, nk_size_t rows, nk_size_t columns, nk_size_t depth,
-    nk_size_t a_stride_elements, nk_size_t c_stride_elements) {
+    nk_size_t a_stride_elements, nk_size_t c_stride_elements) NK_STREAMING_ {
 
     // Read header
     nk_dots_sme_packed_header_t const *header = (nk_dots_sme_packed_header_t const *)b_packed;
@@ -1296,14 +1299,16 @@ __arm_locally_streaming __arm_new("za") static void nk_dots_packed_f64_smef64_st
     }
 }
 
-NK_PUBLIC void nk_dots_packed_f64_smef64(nk_f64_t const *a, void const *b_packed, nk_f64_t *c, nk_size_t rows,
-                                         nk_size_t columns, nk_size_t depth, nk_size_t a_stride_in_bytes,
-                                         nk_size_t c_stride_in_bytes) {
+NK_PUBLIC void nk_dots_packed_f64_smef64( //
+    nk_f64_t const *a, void const *b_packed, nk_f64_t *c, nk_size_t rows, nk_size_t columns, nk_size_t depth,
+    nk_size_t a_stride_in_bytes, nk_size_t c_stride_in_bytes) {
 
     nk_size_t const a_stride_elements = a_stride_in_bytes / sizeof(nk_f64_t);
     nk_size_t const c_stride_elements = c_stride_in_bytes / sizeof(nk_f64_t);
 
+    nk_sme_start_streaming_();
     nk_dots_packed_f64_smef64_streaming_(a, b_packed, c, rows, columns, depth, a_stride_elements, c_stride_elements);
+    nk_sme_stop_streaming_();
 }
 
 #pragma endregion F64 Floats

package/include/numkong/dots.h

@@ -681,6 +681,37 @@ NK_PUBLIC void nk_dots_symmetric_u8_sapphireamx(nk_u8_t const *vectors, nk_size_
                                                 nk_size_t row_start, nk_size_t row_count);
 #endif // NK_TARGET_SAPPHIREAMX
 
+/*  Granite Rapids backends using Intel AMX-FP16 (Advanced Matrix Extensions with FP16 support).
+ *  AMX-FP16 adds TDPFP16PS (FP16×FP16→FP32 tile multiply-accumulate), same tile geometry as BF16.
+ *  The F32 Ozaki kernel splits F32 inputs into 2 FP16 halves for ~35-40 bit effective precision.
+ */
+#if NK_TARGET_GRANITEAMX
+/** @copydoc nk_dots_packed_size_f16 */
+NK_PUBLIC nk_size_t nk_dots_packed_size_f16_graniteamx(nk_size_t width, nk_size_t depth);
+/** @copydoc nk_dots_pack_f16 */
+NK_PUBLIC void nk_dots_pack_f16_graniteamx(nk_f16_t const *b, nk_size_t width, nk_size_t depth, nk_size_t b_stride,
+                                           void *b_packed);
+/** @copydoc nk_dots_packed_f16 */
+NK_PUBLIC void nk_dots_packed_f16_graniteamx(nk_f16_t const *a, void const *b_packed, nk_f32_t *c, nk_size_t height,
+                                             nk_size_t width, nk_size_t depth, nk_size_t a_stride, nk_size_t c_stride);
+/** @copydoc nk_dots_symmetric_f16 */
+NK_PUBLIC void nk_dots_symmetric_f16_graniteamx(nk_f16_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
+                                                nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
+                                                nk_size_t row_start, nk_size_t row_count);
+/** @copydoc nk_dots_packed_size_f16 */
+NK_PUBLIC nk_size_t nk_dots_packed_size_e5m2_graniteamx(nk_size_t width, nk_size_t depth);
+/** @copydoc nk_dots_pack_f16 */
+NK_PUBLIC void nk_dots_pack_e5m2_graniteamx(nk_e5m2_t const *b, nk_size_t width, nk_size_t depth, nk_size_t b_stride,
+                                            void *b_packed);
+/** @copydoc nk_dots_packed_f16 */
+NK_PUBLIC void nk_dots_packed_e5m2_graniteamx(nk_e5m2_t const *a, void const *b_packed, nk_f32_t *c, nk_size_t height,
+                                              nk_size_t width, nk_size_t depth, nk_size_t a_stride, nk_size_t c_stride);
+/** @copydoc nk_dots_symmetric_f16 */
+NK_PUBLIC void nk_dots_symmetric_e5m2_graniteamx(nk_e5m2_t const *vectors, nk_size_t vectors_count, nk_size_t depth,
+                                                 nk_size_t stride, nk_f32_t *result, nk_size_t result_stride,
+                                                 nk_size_t row_start, nk_size_t row_count);
+#endif // NK_TARGET_GRANITEAMX
+
 /*  ARM SME backends using Scalable Matrix Extension.
  *  SME provides ZA tile registers for outer product operations.
  *  F16/BF16/I8/U8/E4M3 use ZA32 tiles, F32/F64 use ZA64 tiles (FEAT_SME_F64F64).
@@ -1858,6 +1889,7 @@ NK_PUBLIC void nk_dots_symmetric_u1_loongsonasx(nk_u1x8_t const *vectors, nk_siz
 #include "numkong/dots/genoa.h"
 #include "numkong/dots/diamond.h"
 #include "numkong/dots/sapphireamx.h"
+#include "numkong/dots/graniteamx.h"
 #include "numkong/dots/neon.h"
 #include "numkong/dots/neonsdot.h"
 #include "numkong/dots/neonfhm.h"
@@ -2002,7 +2034,9 @@ NK_PUBLIC void nk_dots_packed_f64(nk_f64_t const *a, void const *b_packed, nk_f6
 }
 
 NK_PUBLIC nk_size_t nk_dots_packed_size_f16(nk_size_t width, nk_size_t depth) {
-#if NK_TARGET_SME
+#if NK_TARGET_GRANITEAMX
+    return nk_dots_packed_size_f16_graniteamx(width, depth);
+#elif NK_TARGET_SME
     return nk_dots_packed_size_f16_sme(width, depth);
 #elif NK_TARGET_NEONFHM
     return nk_dots_packed_size_f16_neonfhm(width, depth);
@@ -2023,7 +2057,9 @@ NK_PUBLIC nk_size_t nk_dots_packed_size_f16(nk_size_t width, nk_size_t depth) {
 
 NK_PUBLIC void nk_dots_pack_f16(nk_f16_t const *b, nk_size_t width, nk_size_t depth, nk_size_t b_stride,
                                 void *b_packed) {
-#if NK_TARGET_SME
+#if NK_TARGET_GRANITEAMX
+    nk_dots_pack_f16_graniteamx(b, width, depth, b_stride, b_packed);
+#elif NK_TARGET_SME
     nk_dots_pack_f16_sme(b, width, depth, b_stride, b_packed);
 #elif NK_TARGET_NEONFHM
     nk_dots_pack_f16_neonfhm(b, width, depth, b_stride, b_packed);
@@ -2044,7 +2080,9 @@ NK_PUBLIC void nk_dots_pack_f16(nk_f16_t const *b, nk_size_t width, nk_size_t de
 
 NK_PUBLIC void nk_dots_packed_f16(nk_f16_t const *a, void const *b_packed, nk_f32_t *c, nk_size_t height,
                                   nk_size_t width, nk_size_t depth, nk_size_t a_stride, nk_size_t c_stride) {
-#if NK_TARGET_SME
+#if NK_TARGET_GRANITEAMX
+    nk_dots_packed_f16_graniteamx(a, b_packed, c, height, width, depth, a_stride, c_stride);
+#elif NK_TARGET_SME
     nk_dots_packed_f16_sme(a, b_packed, c, height, width, depth, a_stride, c_stride);
 #elif NK_TARGET_NEONFHM
     nk_dots_packed_f16_neonfhm(a, b_packed, c, height, width, depth, a_stride, c_stride);

package/include/numkong/each/serial.h

@@ -76,6 +76,29 @@ extern "C" {
         }                                                                                                             \
     }
 
+/*  Keep the serial instantiations below actually scalar, regardless of build type.
+ *  Without this, -O3 + LTO can vectorize or clone the serial kernels under AVX-512
+ *  callers in dispatch_*.c, which wastes binary and breaks the nk_*_serial-as-scalar-oracle
+ *  contract. See dots/serial.h. */
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
+/* Size bias for release. Gated on NDEBUG so Debug builds keep -O0 for stepping. */
+#if defined(NDEBUG)
+#if defined(_MSC_VER)
+#pragma optimize("s", on)
+#elif defined(__clang__)
+#pragma clang attribute push(__attribute__((minsize)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("Os")
+#endif
+#endif
+
 nk_define_each_sum_(f64, f64, nk_assign_from_to_, nk_assign_from_to_)        // nk_each_sum_f64_serial
 nk_define_each_sum_(f32, f32, nk_assign_from_to_, nk_assign_from_to_)        // nk_each_sum_f32_serial
 nk_define_each_sum_(f16, f32, nk_f16_to_f32_serial, nk_f32_to_f16_serial)    // nk_each_sum_f16_serial
@@ -253,6 +276,22 @@ NK_PUBLIC void nk_each_fma_f64c_serial(nk_f64c_t const *a, nk_f64c_t const *b, n
     }
 }
 
+#if defined(NDEBUG)
+#if defined(_MSC_VER)
+#pragma optimize("", on)
+#elif defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
 #if defined(__cplusplus)
 } // extern "C"
 #endif

package/include/numkong/geospatial/haswell.h

@@ -24,7 +24,7 @@
 #if NK_TARGET_HASWELL
 
 #include "numkong/types.h"
-#include "numkong/trigonometry/haswell.h" // `nk_sin_f64x4_haswell_`, `nk_cos_f64x4_haswell_`, `nk_atan2_f64x4_haswell_`, etc.
+#include "numkong/trigonometry/haswell.h" // `nk_sin_f64x4_haswell_`, `nk_cos_f64x4_haswell_`, `nk_atan2_f64x4_haswell_`
 
 #if defined(__cplusplus)
 extern "C" {

package/include/numkong/geospatial/neon.h

@@ -21,7 +21,7 @@
 #if NK_TARGET_NEON
 
 #include "numkong/types.h"
-#include "numkong/trigonometry/neon.h" // `nk_sin_f64x2_neon_`, `nk_cos_f64x2_neon_`, `nk_atan2_f64x2_neon_`, etc.
+#include "numkong/trigonometry/neon.h" // `nk_sin_f64x2_neon_`, `nk_cos_f64x2_neon_`, `nk_atan2_f64x2_neon_`
 
 #if defined(__cplusplus)
 extern "C" {

package/include/numkong/geospatial/serial.h

@@ -11,15 +11,20 @@
 
 #include "numkong/types.h"
 #include "numkong/spatial/serial.h"      // `nk_f64_sqrt_serial`, `nk_f32_sqrt_serial`
-#include "numkong/trigonometry/serial.h" // `nk_f64_sin`, `nk_f64_cos`, `nk_f64_atan2`, etc.
+#include "numkong/trigonometry/serial.h" // `nk_f64_sin`, `nk_f64_cos`, `nk_f64_atan2`
 
 #if defined(__cplusplus)
 extern "C" {
 #endif
 
-/*  Serial implementations of geospatial distance functions.
- *  These use the trigonometric functions from trigonometry.h for sin, cos, and atan2.
- */
+/*  Keep the serial instantiations below actually scalar, regardless of build type.
+ *  See dots/serial.h for rationale. */
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
 
 NK_PUBLIC void nk_haversine_f64_serial(             //
     nk_f64_t const *a_lats, nk_f64_t const *a_lons, //
@@ -302,6 +307,12 @@ NK_PUBLIC void nk_vincenty_f32_serial(              //
     }
 }
 
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
 #if defined(__cplusplus)
 } // extern "C"
 #endif

package/include/numkong/geospatial/skylake.h

@@ -24,7 +24,7 @@
 #if NK_TARGET_SKYLAKE
 
 #include "numkong/types.h"
-#include "numkong/trigonometry/skylake.h" // `nk_sin_f64x8_skylake_`, `nk_cos_f64x8_skylake_`, `nk_atan2_f64x8_skylake_`, etc.
+#include "numkong/trigonometry/skylake.h" // `nk_sin_f64x8_skylake_`, `nk_cos_f64x8_skylake_`, `nk_atan2_f64x8_skylake_`
 
 #if defined(__cplusplus)
 extern "C" {

package/include/numkong/maxsim/serial.h

@@ -71,6 +71,15 @@ NK_STATIC_ASSERT(sizeof(nk_maxsim_vector_metadata_t) == 12, nk_maxsim_vector_met
  */
 typedef void (*nk_maxsim_to_f32_t)(void const *source, nk_f32_t *destination);
 
+/*  Keep the serial instantiations below actually scalar, regardless of build type.
+ *  See dots/serial.h for rationale. */
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
 /** @brief Identity conversion for f32 sources — just a typed memcpy. */
 NK_INTERNAL void nk_f32_to_f32_(void const *source, nk_f32_t *destination) { *destination = *(nk_f32_t const *)source; }
 
@@ -483,6 +492,12 @@ NK_PUBLIC void nk_maxsim_packed_f16_serial( //
     *result = (nk_f32_t)total_angular_distance;
 }
 
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
 #if defined(__cplusplus)
 } // extern "C"
 #endif

package/include/numkong/maxsim/sme.h

@@ -46,7 +46,8 @@
 #if NK_TARGET_ARM64_
 #if NK_TARGET_SME
 
-#include "numkong/dots/sme.h" // nk_dots_sme_packed_header_t, nk_dots_pack_{f16,bf16}_sme, nk_dots_packed_size_{f16,bf16}_sme
+#include "numkong/dots/sme.h"   // `nk_dots_sme_packed_header_t`
+#include "numkong/reduce/sve.h" // `nk_svaddv_f64_`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -90,10 +91,9 @@ NK_STATIC_ASSERT(sizeof(nk_maxsim_sme_packed_header_t) == 64, nk_maxsim_sme_pack
  *
  *  1-tile remainder: uses ZA0 only, with predicated loads for partial tiles.
  */
-__arm_locally_streaming __arm_new("za") static void nk_maxsim_packed_f16_streaming_( //
-    void const *query_packed, void const *document_packed,                           //
-    nk_size_t query_count, nk_size_t document_count,                                 //
-    nk_size_t depth, nk_f32_t *result) {
+__arm_new("za") static void nk_maxsim_packed_f16_streaming_( //
+    void const *query_packed, void const *document_packed, nk_size_t query_count, nk_size_t document_count,
+    nk_size_t depth, nk_f32_t *result) NK_STREAMING_ {
 
     nk_maxsim_sme_packed_header_t const *query_header = (nk_maxsim_sme_packed_header_t const *)query_packed;
     nk_maxsim_sme_packed_header_t const *document_header = (nk_maxsim_sme_packed_header_t const *)document_packed;
@@ -258,18 +258,19 @@ __arm_locally_streaming __arm_new("za") static void nk_maxsim_packed_f16_streami
             document_inverse_norms_f32x);
         svfloat32_t angular_distance_f32x = svmax_f32_x(
             row_predicate_b32x, svsub_f32_x(row_predicate_b32x, svdup_f32(1.0f), cosine_f32x), svdup_f32(0.0f));
-        total_angular_distance += svaddv_f32(row_predicate_b32x, angular_distance_f32x);
+        total_angular_distance += nk_svaddv_f32_(row_predicate_b32x, angular_distance_f32x);
     }
 
     *result = total_angular_distance;
 }
 
-NK_PUBLIC void nk_maxsim_packed_f16_sme(                              //
-    void const *query_packed, void const *document_packed,            //
-    nk_size_t query_count, nk_size_t document_count, nk_size_t depth, //
-    nk_f32_t *result) {                                               //
+NK_PUBLIC void nk_maxsim_packed_f16_sme( //
+    void const *query_packed, void const *document_packed, nk_size_t query_count, nk_size_t document_count,
+    nk_size_t depth, nk_f32_t *result) {
 
+    nk_sme_start_streaming_();
     nk_maxsim_packed_f16_streaming_(query_packed, document_packed, query_count, document_count, depth, result);
+    nk_sme_stop_streaming_();
 }
 
 /**
@@ -281,10 +282,9 @@ NK_PUBLIC void nk_maxsim_packed_f16_sme(                              //
  *
  *  1-tile remainder: uses ZA0 only, with predicated loads for partial tiles.
  */
-__arm_locally_streaming __arm_new("za") static void nk_maxsim_packed_bf16_streaming_( //
-    void const *query_packed, void const *document_packed,                            //
-    nk_size_t query_count, nk_size_t document_count,                                  //
-    nk_size_t depth, nk_f32_t *result) {
+__arm_new("za") static void nk_maxsim_packed_bf16_streaming_( //
+    void const *query_packed, void const *document_packed, nk_size_t query_count, nk_size_t document_count,
+    nk_size_t depth, nk_f32_t *result) NK_STREAMING_ {
 
     nk_maxsim_sme_packed_header_t const *query_header = (nk_maxsim_sme_packed_header_t const *)query_packed;
     nk_maxsim_sme_packed_header_t const *document_header = (nk_maxsim_sme_packed_header_t const *)document_packed;
@@ -454,18 +454,19 @@ __arm_locally_streaming __arm_new("za") static void nk_maxsim_packed_bf16_stream
             document_inverse_norms_f32x);
         svfloat32_t angular_distance_f32x = svmax_f32_x(
             row_predicate_b32x, svsub_f32_x(row_predicate_b32x, svdup_f32(1.0f), cosine_f32x), svdup_f32(0.0f));
-        total_angular_distance += svaddv_f32(row_predicate_b32x, angular_distance_f32x);
+        total_angular_distance += nk_svaddv_f32_(row_predicate_b32x, angular_distance_f32x);
     }
 
     *result = total_angular_distance;
 }
 
-NK_PUBLIC void nk_maxsim_packed_bf16_sme(                             //
-    void const *query_packed, void const *document_packed,            //
-    nk_size_t query_count, nk_size_t document_count, nk_size_t depth, //
-    nk_f32_t *result) {                                               //
+NK_PUBLIC void nk_maxsim_packed_bf16_sme( //
+    void const *query_packed, void const *document_packed, nk_size_t query_count, nk_size_t document_count,
+    nk_size_t depth, nk_f32_t *result) {
 
+    nk_sme_start_streaming_();
     nk_maxsim_packed_bf16_streaming_(query_packed, document_packed, query_count, document_count, depth, result);
+    nk_sme_stop_streaming_();
 }
 
 NK_PUBLIC nk_size_t nk_maxsim_packed_size_bf16_sme(nk_size_t columns, nk_size_t depth) { //
@@ -649,7 +650,7 @@ NK_PUBLIC nk_f64_t nk_maxsim_reduce_dot_f32_ssve_(                         //
         svfloat64_t b_odd_f64x = svcvtlt_f64_f32_x(predicate_odd_b64x, b_f32x);
         accumulator_odd_f64x = svmla_f64_m(predicate_odd_b64x, accumulator_odd_f64x, a_odd_f64x, b_odd_f64x);
     }
-    return svaddv_f64(svptrue_b64(), accumulator_even_f64x) + svaddv_f64(svptrue_b64(), accumulator_odd_f64x);
+    return nk_svaddv_f64_(svptrue_b64(), accumulator_even_f64x) + nk_svaddv_f64_(svptrue_b64(), accumulator_odd_f64x);
 }
 
 /**
@@ -687,7 +688,7 @@ NK_PUBLIC nk_f64_t nk_maxsim_angular_from_dots_ssve_(
         svfloat64_t angular_distance_f64x = svsub_f64_x(predicate_b64x, svdup_f64(1.0), cosine_f64x);
         angular_distance_f64x = svmax_f64_x(predicate_b64x, angular_distance_f64x, svdup_f64(0.0));
 
-        total_angular_distance_f64 += svaddv_f64(predicate_b64x, angular_distance_f64x);
+        total_angular_distance_f64 += nk_svaddv_f64_(predicate_b64x, angular_distance_f64x);
     }
     return total_angular_distance_f64;
 }
@@ -701,10 +702,9 @@ NK_PUBLIC nk_f64_t nk_maxsim_angular_from_dots_ssve_(
  *  Refinement: tile-wide interleaved f64 dot products for the winning (query, document) pairs.
  *  Angular distance: 1 - dot / sqrt(||q||^2 * ||d||^2), accumulated with f64.
  */
-__arm_locally_streaming __arm_new("za") static void nk_maxsim_packed_f32_streaming_( //
-    void const *query_packed, void const *document_packed,                           //
-    nk_size_t query_count, nk_size_t document_count, nk_size_t depth,                //
-    nk_f64_t *result) {
+__arm_new("za") static void nk_maxsim_packed_f32_streaming_( //
+    void const *query_packed, void const *document_packed, nk_size_t query_count, nk_size_t document_count,
+    nk_size_t depth, nk_f64_t *result) NK_STREAMING_ {
 
     nk_maxsim_sme_packed_header_t const *query_header = (nk_maxsim_sme_packed_header_t const *)query_packed;
     nk_maxsim_sme_packed_header_t const *document_header = (nk_maxsim_sme_packed_header_t const *)document_packed;
@@ -937,10 +937,10 @@ __arm_locally_streaming __arm_new("za") static void nk_maxsim_packed_f32_streami
 
             // Reduce SVE accumulators to scalars and compute angular distances
             nk_f64_t dot_products_f64[4];
-            dot_products_f64[0] = svaddv_f64(svptrue_b64(), accumulator_0_f64x);
-            dot_products_f64[1] = svaddv_f64(svptrue_b64(), accumulator_1_f64x);
-            dot_products_f64[2] = svaddv_f64(svptrue_b64(), accumulator_2_f64x);
-            dot_products_f64[3] = svaddv_f64(svptrue_b64(), accumulator_3_f64x);
+            dot_products_f64[0] = nk_svaddv_f64_(svptrue_b64(), accumulator_0_f64x);
+            dot_products_f64[1] = nk_svaddv_f64_(svptrue_b64(), accumulator_1_f64x);
+            dot_products_f64[2] = nk_svaddv_f64_(svptrue_b64(), accumulator_2_f64x);
+            dot_products_f64[3] = nk_svaddv_f64_(svptrue_b64(), accumulator_3_f64x);
             nk_f64_t batch_query_norms_f64[4], batch_document_norms_f64[4];
             for (nk_size_t i = 0; i < 4; i++) {
                 batch_query_norms_f64[i] = (nk_f64_t)query_norms[row_start + row_batch_start + i];
@@ -969,12 +969,13 @@ __arm_locally_streaming __arm_new("za") static void nk_maxsim_packed_f32_streami
     *result = total_angular_distance_f64;
 }
 
-NK_PUBLIC void nk_maxsim_packed_f32_sme(                              //
-    void const *query_packed, void const *document_packed,            //
-    nk_size_t query_count, nk_size_t document_count, nk_size_t depth, //
-    nk_f64_t *result) {                                               //
+NK_PUBLIC void nk_maxsim_packed_f32_sme( //
+    void const *query_packed, void const *document_packed, nk_size_t query_count, nk_size_t document_count,
+    nk_size_t depth, nk_f64_t *result) {
 
+    nk_sme_start_streaming_();
     nk_maxsim_packed_f32_streaming_(query_packed, document_packed, query_count, document_count, depth, result);
+    nk_sme_stop_streaming_();
 }
 
 #if defined(__clang__)