numkong 7.4.4 → 7.5.0

package/include/numkong/curved/smef64.h

@@ -52,9 +52,10 @@
 #if NK_TARGET_SMEF64
 
 #include "numkong/types.h"
+#include "numkong/reduce/sve.h"    // `nk_svaddv_f64_`
 #include "numkong/spatial/neon.h"  // `nk_f64_sqrt_neon`
-#include "numkong/dots/sme.h"      // nk_sme_zero_za64_tile_0_, etc. (for f32 FMOPA)
-#include "numkong/curved/serial.h" // `nk_bilinear_f64_serial`, etc.
+#include "numkong/dots/sme.h"      // `nk_sme_zero_za64_tile_0_`
+#include "numkong/curved/serial.h" // `nk_bilinear_f64_serial`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -90,8 +91,8 @@ NK_PUBLIC void nk_dot2_f64_sve_accumulate_(svbool_t predicate_b64x, svfloat64_t
  *  @brief f32 bilinear: GEMV via FMOPA (widening f32→f64, exact accumulation).
  *  ZA0.D = C staging, ZA1.D = GEMV accumulator.
  */
-__arm_locally_streaming __arm_new("za") static void nk_bilinear_f32_smef64_streaming_(
-    nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t dimensions, nk_f64_t *result) {
+__arm_new("za") static void nk_bilinear_f32_smef64_streaming_( //
+    nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t dimensions, nk_f64_t *result) NK_STREAMING_ {
     svbool_t predicate_body_b64x = svptrue_b64();
     nk_size_t tile_dimension = svcntd();
     nk_f64_t outer_sum_f64 = 0.0;
@@ -124,24 +125,25 @@ __arm_locally_streaming __arm_new("za") static void nk_bilinear_f32_smef64_strea
         svfloat64_t v_f64x = svread_ver_za64_f64_m(svdup_f64(0.0), row_predicate_b64x, 1, 0);
         svfloat64_t a_f64x = svcvt_f64_f32_x(
             row_predicate_b64x, svreinterpret_f32_u64(svld1uw_u64(row_predicate_b64x, (nk_u32_t const *)(a + row))));
-        outer_sum_f64 += svaddv_f64(predicate_body_b64x, svmul_f64_x(row_predicate_b64x, a_f64x, v_f64x));
+        outer_sum_f64 += nk_svaddv_f64_(predicate_body_b64x, svmul_f64_x(row_predicate_b64x, a_f64x, v_f64x));
     }
 
     *result = outer_sum_f64;
 }
 
-NK_PUBLIC void nk_bilinear_f32_smef64(nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t dimensions,
-                                      nk_f64_t *result) {
+NK_PUBLIC void nk_bilinear_f32_smef64( //
+    nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t dimensions, nk_f64_t *result) {
+    nk_sme_start_streaming_();
     nk_bilinear_f32_smef64_streaming_(a, b, c, dimensions, result);
+    nk_sme_stop_streaming_();
 }
 
 /**
  *  @brief f32 Mahalanobis: GEMV v = C×d via FMOPA, where d = a − b (exact in f64).
  *  ZA0.D = C staging, ZA1.D = GEMV accumulator.
  */
-__arm_locally_streaming __arm_new("za") static nk_f64_t
-    nk_mahalanobis_f32_smef64_streaming_(nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c,
-                                         nk_size_t dimensions) {
+__arm_new("za") static nk_f64_t nk_mahalanobis_f32_smef64_streaming_( //
+    nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t dimensions) NK_STREAMING_ {
 
     svbool_t predicate_body_b64x = svptrue_b64();
     nk_size_t tile_dimension = svcntd();
@@ -179,15 +181,17 @@ __arm_locally_streaming __arm_new("za") static nk_f64_t
         svfloat64_t b_f64x = svcvt_f64_f32_x(
             row_predicate_b64x, svreinterpret_f32_u64(svld1uw_u64(row_predicate_b64x, (nk_u32_t const *)(b + row))));
         svfloat64_t d_f64x = svsub_f64_x(row_predicate_b64x, a_f64x, b_f64x);
-        outer_sum_f64 += svaddv_f64(predicate_body_b64x, svmul_f64_x(row_predicate_b64x, d_f64x, v_f64x));
+        outer_sum_f64 += nk_svaddv_f64_(predicate_body_b64x, svmul_f64_x(row_predicate_b64x, d_f64x, v_f64x));
     }
 
     return outer_sum_f64;
 }
 
-NK_PUBLIC void nk_mahalanobis_f32_smef64(nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t dimensions,
-                                         nk_f64_t *result) {
+NK_PUBLIC void nk_mahalanobis_f32_smef64( //
+    nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t dimensions, nk_f64_t *result) {
+    nk_sme_start_streaming_();
     nk_f64_t quadratic = nk_mahalanobis_f32_smef64_streaming_(a, b, c, dimensions);
+    nk_sme_stop_streaming_();
     *result = nk_f64_sqrt_neon(quadratic > 0 ? quadratic : 0);
 }
 
@@ -195,9 +199,8 @@ NK_PUBLIC void nk_mahalanobis_f32_smef64(nk_f32_t const *a, nk_f32_t const *b, n
  *  @brief f64 bilinear: row-by-row streaming SVE with Dot2 compensation.
  *  4-row fast path shares b_f64x loads; 1-row tail for remainder.
  */
-__arm_locally_streaming static void nk_bilinear_f64_smef64_streaming_(nk_f64_t const *a, nk_f64_t const *b,
-                                                                      nk_f64_t const *c, nk_size_t dimensions,
-                                                                      nk_f64_t *result) {
+static void nk_bilinear_f64_smef64_ssve_( //
+    nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t dimensions, nk_f64_t *result) NK_STREAMING_ {
     svbool_t predicate_all_b64x = svptrue_b64();
     nk_f64_t outer_sum = 0.0, outer_comp = 0.0;
     nk_size_t row = 0;
@@ -226,14 +229,18 @@ __arm_locally_streaming static void nk_bilinear_f64_smef64_streaming_(nk_f64_t c
             predicate_b64x = svwhilelt_b64(j, dimensions);
         }
 
-        nk_f64_dot2_(&outer_sum, &outer_comp, a0,
-                     svaddv_f64(predicate_all_b64x, sum_0_f64x) + svaddv_f64(predicate_all_b64x, compensation_0_f64x));
-        nk_f64_dot2_(&outer_sum, &outer_comp, a1,
-                     svaddv_f64(predicate_all_b64x, sum_1_f64x) + svaddv_f64(predicate_all_b64x, compensation_1_f64x));
-        nk_f64_dot2_(&outer_sum, &outer_comp, a2,
-                     svaddv_f64(predicate_all_b64x, sum_2_f64x) + svaddv_f64(predicate_all_b64x, compensation_2_f64x));
-        nk_f64_dot2_(&outer_sum, &outer_comp, a3,
-                     svaddv_f64(predicate_all_b64x, sum_3_f64x) + svaddv_f64(predicate_all_b64x, compensation_3_f64x));
+        nk_f64_dot2_(
+            &outer_sum, &outer_comp, a0,
+            nk_svaddv_f64_(predicate_all_b64x, sum_0_f64x) + nk_svaddv_f64_(predicate_all_b64x, compensation_0_f64x));
+        nk_f64_dot2_(
+            &outer_sum, &outer_comp, a1,
+            nk_svaddv_f64_(predicate_all_b64x, sum_1_f64x) + nk_svaddv_f64_(predicate_all_b64x, compensation_1_f64x));
+        nk_f64_dot2_(
+            &outer_sum, &outer_comp, a2,
+            nk_svaddv_f64_(predicate_all_b64x, sum_2_f64x) + nk_svaddv_f64_(predicate_all_b64x, compensation_2_f64x));
+        nk_f64_dot2_(
+            &outer_sum, &outer_comp, a3,
+            nk_svaddv_f64_(predicate_all_b64x, sum_3_f64x) + nk_svaddv_f64_(predicate_all_b64x, compensation_3_f64x));
     }
 
     // 1-row tail
@@ -250,24 +257,27 @@ __arm_locally_streaming static void nk_bilinear_f64_smef64_streaming_(nk_f64_t c
             predicate_b64x = svwhilelt_b64(j, dimensions);
         }
 
-        nk_f64_t cb_j = svaddv_f64(predicate_all_b64x, sum_f64x) + svaddv_f64(predicate_all_b64x, compensation_f64x);
+        nk_f64_t cb_j = nk_svaddv_f64_(predicate_all_b64x, sum_f64x) +
+                        nk_svaddv_f64_(predicate_all_b64x, compensation_f64x);
         nk_f64_dot2_(&outer_sum, &outer_comp, a[row], cb_j);
     }
 
     *result = outer_sum + outer_comp;
 }
 
-NK_PUBLIC void nk_bilinear_f64_smef64(nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t dimensions,
-                                      nk_f64_t *result) {
-    nk_bilinear_f64_smef64_streaming_(a, b, c, dimensions, result);
+NK_PUBLIC void nk_bilinear_f64_smef64( //
+    nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t dimensions, nk_f64_t *result) {
+    nk_sme_start_streaming_();
+    nk_bilinear_f64_smef64_ssve_(a, b, c, dimensions, result);
+    nk_sme_stop_streaming_();
 }
 
 /**
  *  @brief f64 Mahalanobis: row-by-row streaming SVE with Dot2 compensation.
  *  4-row fast path shares (a−b) column vector; 1-row tail for remainder.
  */
-__arm_locally_streaming static nk_f64_t nk_mahalanobis_f64_smef64_streaming_(nk_f64_t const *a, nk_f64_t const *b,
-                                                                             nk_f64_t const *c, nk_size_t dimensions) {
+static nk_f64_t nk_mahalanobis_f64_smef64_ssve_( //
+    nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t dimensions) NK_STREAMING_ {
     svbool_t predicate_all_b64x = svptrue_b64();
     nk_f64_t outer_sum = 0.0, outer_comp = 0.0;
     nk_size_t row = 0;
@@ -298,14 +308,18 @@ __arm_locally_streaming static nk_f64_t nk_mahalanobis_f64_smef64_streaming_(nk_
             predicate_b64x = svwhilelt_b64(j, dimensions);
         }
 
-        nk_f64_dot2_(&outer_sum, &outer_comp, d0,
-                     svaddv_f64(predicate_all_b64x, sum_0_f64x) + svaddv_f64(predicate_all_b64x, compensation_0_f64x));
-        nk_f64_dot2_(&outer_sum, &outer_comp, d1,
-                     svaddv_f64(predicate_all_b64x, sum_1_f64x) + svaddv_f64(predicate_all_b64x, compensation_1_f64x));
-        nk_f64_dot2_(&outer_sum, &outer_comp, d2,
-                     svaddv_f64(predicate_all_b64x, sum_2_f64x) + svaddv_f64(predicate_all_b64x, compensation_2_f64x));
-        nk_f64_dot2_(&outer_sum, &outer_comp, d3,
-                     svaddv_f64(predicate_all_b64x, sum_3_f64x) + svaddv_f64(predicate_all_b64x, compensation_3_f64x));
+        nk_f64_dot2_(
+            &outer_sum, &outer_comp, d0,
+            nk_svaddv_f64_(predicate_all_b64x, sum_0_f64x) + nk_svaddv_f64_(predicate_all_b64x, compensation_0_f64x));
+        nk_f64_dot2_(
+            &outer_sum, &outer_comp, d1,
+            nk_svaddv_f64_(predicate_all_b64x, sum_1_f64x) + nk_svaddv_f64_(predicate_all_b64x, compensation_1_f64x));
+        nk_f64_dot2_(
+            &outer_sum, &outer_comp, d2,
+            nk_svaddv_f64_(predicate_all_b64x, sum_2_f64x) + nk_svaddv_f64_(predicate_all_b64x, compensation_2_f64x));
+        nk_f64_dot2_(
+            &outer_sum, &outer_comp, d3,
+            nk_svaddv_f64_(predicate_all_b64x, sum_3_f64x) + nk_svaddv_f64_(predicate_all_b64x, compensation_3_f64x));
     }
 
     // 1-row tail
@@ -324,16 +338,19 @@ __arm_locally_streaming static nk_f64_t nk_mahalanobis_f64_smef64_streaming_(nk_
             predicate_b64x = svwhilelt_b64(j, dimensions);
         }
 
-        nk_f64_t cb_j = svaddv_f64(predicate_all_b64x, sum_f64x) + svaddv_f64(predicate_all_b64x, compensation_f64x);
+        nk_f64_t cb_j = nk_svaddv_f64_(predicate_all_b64x, sum_f64x) +
+                        nk_svaddv_f64_(predicate_all_b64x, compensation_f64x);
         nk_f64_dot2_(&outer_sum, &outer_comp, diff_row, cb_j);
     }
 
     return outer_sum + outer_comp;
 }
 
-NK_PUBLIC void nk_mahalanobis_f64_smef64(nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t dimensions,
-                                         nk_f64_t *result) {
-    nk_f64_t quadratic = nk_mahalanobis_f64_smef64_streaming_(a, b, c, dimensions);
+NK_PUBLIC void nk_mahalanobis_f64_smef64( //
+    nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t dimensions, nk_f64_t *result) {
+    nk_sme_start_streaming_();
+    nk_f64_t quadratic = nk_mahalanobis_f64_smef64_ssve_(a, b, c, dimensions);
+    nk_sme_stop_streaming_();
     *result = nk_f64_sqrt_neon(quadratic > 0 ? quadratic : 0);
 }
 
@@ -341,11 +358,9 @@ NK_PUBLIC void nk_mahalanobis_f64_smef64(nk_f64_t const *a, nk_f64_t const *b, n
  *  @brief f32c bilinear: complex GEMV via FMOPA (widening f32→f64).
  *  ZA0.D = C staging, ZA1.D = v_real accumulator, ZA2.D = v_imag accumulator.
  */
-__arm_locally_streaming __arm_new("za") static void nk_bilinear_f32c_smef64_streaming_(nk_f32c_t const *a_pairs,
-                                                                                       nk_f32c_t const *b_pairs,
-                                                                                       nk_f32c_t const *c_pairs,
-                                                                                       nk_size_t dimensions,
-                                                                                       nk_f64c_t *results) {
+__arm_new("za") static void nk_bilinear_f32c_smef64_streaming_( //
+    nk_f32c_t const *a_pairs, nk_f32c_t const *b_pairs, nk_f32c_t const *c_pairs, nk_size_t dimensions,
+    nk_f64c_t *results) NK_STREAMING_ {
     svbool_t predicate_body_b64x = svptrue_b64();
     nk_size_t tile_dimension = svcntd();
     nk_f64_t outer_sum_real_f64 = 0.0, outer_sum_imag_f64 = 0.0;
@@ -407,10 +422,10 @@ __arm_locally_streaming __arm_new("za") static void nk_bilinear_f32c_smef64_stre
         svfloat64_t a_im_f64x = svcvt_f64_f32_x(row_predicate_b64x, svtrn2_f32(a_f32x, a_f32x));
 
         // Complex dot: a × v
-        outer_sum_real_f64 += svaddv_f64(
+        outer_sum_real_f64 += nk_svaddv_f64_(
             predicate_body_b64x, svsub_f64_x(row_predicate_b64x, svmul_f64_x(row_predicate_b64x, a_re_f64x, v_re_f64x),
                                              svmul_f64_x(row_predicate_b64x, a_im_f64x, v_im_f64x)));
-        outer_sum_imag_f64 += svaddv_f64(
+        outer_sum_imag_f64 += nk_svaddv_f64_(
             predicate_body_b64x, svadd_f64_x(row_predicate_b64x, svmul_f64_x(row_predicate_b64x, a_re_f64x, v_im_f64x),
                                              svmul_f64_x(row_predicate_b64x, a_im_f64x, v_re_f64x)));
     }
@@ -419,19 +434,21 @@ __arm_locally_streaming __arm_new("za") static void nk_bilinear_f32c_smef64_stre
     results->imag = outer_sum_imag_f64;
 }
 
-NK_PUBLIC void nk_bilinear_f32c_smef64(nk_f32c_t const *a_pairs, nk_f32c_t const *b_pairs, nk_f32c_t const *c_pairs,
-                                       nk_size_t dimensions, nk_f64c_t *results) {
+NK_PUBLIC void nk_bilinear_f32c_smef64( //
+    nk_f32c_t const *a_pairs, nk_f32c_t const *b_pairs, nk_f32c_t const *c_pairs, nk_size_t dimensions,
+    nk_f64c_t *results) {
+    nk_sme_start_streaming_();
     nk_bilinear_f32c_smef64_streaming_(a_pairs, b_pairs, c_pairs, dimensions, results);
+    nk_sme_stop_streaming_();
 }
 
 /**
  *  @brief f64c bilinear: interleaved Dot2 with permute + deferred XOR sign-flip.
  *  2 accumulators instead of 4, halving inner loop work (~15 vs ~28 SVE ops).
  */
-__arm_locally_streaming static void nk_bilinear_f64c_smef64_streaming_(nk_f64c_t const *a_pairs,
-                                                                       nk_f64c_t const *b_pairs,
-                                                                       nk_f64c_t const *c_pairs, nk_size_t dimensions,
-                                                                       nk_f64c_t *results) {
+static void nk_bilinear_f64c_smef64_ssve_( //
+    nk_f64c_t const *a_pairs, nk_f64c_t const *b_pairs, nk_f64c_t const *c_pairs, nk_size_t dimensions,
+    nk_f64c_t *results) NK_STREAMING_ {
     svbool_t predicate_all_b64x = svptrue_b64();
     nk_f64_t outer_sum_real = 0.0, outer_comp_real = 0.0;
     nk_f64_t outer_sum_imag = 0.0, outer_comp_imag = 0.0;
@@ -474,10 +491,10 @@ __arm_locally_streaming static void nk_bilinear_f64c_smef64_streaming_(nk_f64c_t
             sveor_u64_x(predicate_all_b64x, svreinterpret_u64_f64(sum_real_f64x), sign_mask_u64x));
         comp_real_f64x = svreinterpret_f64_u64(
             sveor_u64_x(predicate_all_b64x, svreinterpret_u64_f64(comp_real_f64x), sign_mask_u64x));
-        nk_f64_t inner_real = svaddv_f64(predicate_all_b64x,
-                                         svadd_f64_x(predicate_all_b64x, sum_real_f64x, comp_real_f64x));
-        nk_f64_t inner_imag = svaddv_f64(predicate_all_b64x,
-                                         svadd_f64_x(predicate_all_b64x, sum_imag_f64x, comp_imag_f64x));
+        nk_f64_t inner_real = nk_svaddv_f64_(predicate_all_b64x,
+                                             svadd_f64_x(predicate_all_b64x, sum_real_f64x, comp_real_f64x));
+        nk_f64_t inner_imag = nk_svaddv_f64_(predicate_all_b64x,
+                                             svadd_f64_x(predicate_all_b64x, sum_imag_f64x, comp_imag_f64x));
 
         // Outer Dot2 complex multiply: a × inner
         nk_f64_dot2_(&outer_sum_real, &outer_comp_real, a_real, inner_real);
@@ -490,9 +507,12 @@ __arm_locally_streaming static void nk_bilinear_f64c_smef64_streaming_(nk_f64c_t
     results->imag = outer_sum_imag + outer_comp_imag;
 }
 
-NK_PUBLIC void nk_bilinear_f64c_smef64(nk_f64c_t const *a_pairs, nk_f64c_t const *b_pairs, nk_f64c_t const *c_pairs,
-                                       nk_size_t dimensions, nk_f64c_t *results) {
-    nk_bilinear_f64c_smef64_streaming_(a_pairs, b_pairs, c_pairs, dimensions, results);
+NK_PUBLIC void nk_bilinear_f64c_smef64( //
+    nk_f64c_t const *a_pairs, nk_f64c_t const *b_pairs, nk_f64c_t const *c_pairs, nk_size_t dimensions,
+    nk_f64c_t *results) {
+    nk_sme_start_streaming_();
+    nk_bilinear_f64c_smef64_ssve_(a_pairs, b_pairs, c_pairs, dimensions, results);
+    nk_sme_stop_streaming_();
 }
 
 #if defined(__clang__)

package/include/numkong/dot/rvvbf16.h

@@ -22,7 +22,7 @@
 #if NK_TARGET_RVVBF16
 
 #include "numkong/types.h"
-#include "numkong/cast/rvv.h" // `nk_e4m3m1_to_bf16m2_rvv_`, `nk_e5m2m1_to_bf16m2_rvv_`, etc.
+#include "numkong/cast/rvv.h" // `nk_e4m3m1_to_bf16m2_rvv_`, `nk_e5m2m1_to_bf16m2_rvv_`
 
 #if defined(__clang__)
 #pragma clang attribute push(__attribute__((target("arch=+v,+zvfbfwma"))), apply_to = function)

package/include/numkong/dot/rvvhalf.h

@@ -23,7 +23,7 @@
 #if NK_TARGET_RVVHALF
 
 #include "numkong/types.h"
-#include "numkong/cast/rvv.h" // `nk_e4m3m1_to_f16m2_rvv_`, `nk_e2m3m1_to_f16m2_rvv_`, etc.
+#include "numkong/cast/rvv.h" // `nk_e4m3m1_to_f16m2_rvv_`, `nk_e2m3m1_to_f16m2_rvv_`
 
 #if defined(__clang__)
 #pragma clang attribute push(__attribute__((target("arch=+v,+zvfh"))), apply_to = function)

package/include/numkong/dot/sve.h

@@ -39,6 +39,7 @@
 #if NK_TARGET_SVE
 
 #include "numkong/types.h"      // `nk_f32_t`
+#include "numkong/reduce/sve.h" // `nk_svaddv_f64_`
 #include "numkong/dot/serial.h" // `nk_u1x8_popcount_`
 
 #if defined(__cplusplus)
@@ -110,7 +111,7 @@ NK_PUBLIC void nk_dot_f32_sve(nk_f32_t const *a_scalars, nk_f32_t const *b_scala
         ab_f64x = svmla_f64_m(pred_odd_b64x, ab_f64x, svcvt_f64_f32_x(pred_odd_b64x, svext_f32(a_f32x, a_f32x, 1)),
                               svcvt_f64_f32_x(pred_odd_b64x, svext_f32(b_f32x, b_f32x, 1)));
     }
-    *result = svaddv_f64(svptrue_b64(), ab_f64x);
+    *result = nk_svaddv_f64_(svptrue_b64(), ab_f64x);
 }
 
 NK_PUBLIC void nk_dot_f32c_sve(nk_f32c_t const *a_pairs, nk_f32c_t const *b_pairs, nk_size_t count_pairs,
@@ -149,8 +150,8 @@ NK_PUBLIC void nk_dot_f32c_sve(nk_f32c_t const *a_pairs, nk_f32c_t const *b_pair
         ab_imag_f64x = svmla_f64_m(pred_odd_b64x, ab_imag_f64x, a_real_odd_f64x, b_imag_odd_f64x);
         ab_imag_f64x = svmla_f64_m(pred_odd_b64x, ab_imag_f64x, a_imag_odd_f64x, b_real_odd_f64x);
     }
-    results->real = svaddv_f64(svptrue_b64(), ab_real_f64x);
-    results->imag = svaddv_f64(svptrue_b64(), ab_imag_f64x);
+    results->real = nk_svaddv_f64_(svptrue_b64(), ab_real_f64x);
+    results->imag = nk_svaddv_f64_(svptrue_b64(), ab_imag_f64x);
 }
 
 NK_PUBLIC void nk_vdot_f32c_sve(nk_f32c_t const *a_pairs, nk_f32c_t const *b_pairs, nk_size_t count_pairs,
@@ -189,8 +190,8 @@ NK_PUBLIC void nk_vdot_f32c_sve(nk_f32c_t const *a_pairs, nk_f32c_t const *b_pai
         ab_imag_f64x = svmla_f64_m(pred_odd_b64x, ab_imag_f64x, a_real_odd_f64x, b_imag_odd_f64x);
         ab_imag_f64x = svmls_f64_m(pred_odd_b64x, ab_imag_f64x, a_imag_odd_f64x, b_real_odd_f64x);
     }
-    results->real = svaddv_f64(svptrue_b64(), ab_real_f64x);
-    results->imag = svaddv_f64(svptrue_b64(), ab_imag_f64x);
+    results->real = nk_svaddv_f64_(svptrue_b64(), ab_real_f64x);
+    results->imag = nk_svaddv_f64_(svptrue_b64(), ab_imag_f64x);
 }
 
 NK_PUBLIC void nk_dot_f64_sve(nk_f64_t const *a_scalars, nk_f64_t const *b_scalars, nk_size_t count_scalars,

package/include/numkong/dot/svebfdot.h

@@ -31,6 +31,7 @@
 #if NK_TARGET_SVEBFDOT
 
 #include "numkong/types.h"
+#include "numkong/reduce/sve.h" // `nk_svaddv_f64_`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -56,7 +57,7 @@ NK_PUBLIC void nk_dot_bf16_svebfdot(nk_bf16_t const *a_scalars, nk_bf16_t const
         sum_f32x = svbfdot_f32(sum_f32x, a_bf16x, b_bf16x);
         idx_scalars += svcnth();
     } while (idx_scalars < count_scalars);
-    *result = svaddv_f32(svptrue_b32(), sum_f32x);
+    *result = nk_svaddv_f32_(svptrue_b32(), sum_f32x);
 }
 
 #if defined(__clang__)

package/include/numkong/dot/svehalf.h

@@ -33,6 +33,7 @@
 #if NK_TARGET_SVEHALF
 
 #include "numkong/types.h"      // `nk_f16_t`
+#include "numkong/reduce/sve.h" // `nk_svaddv_f64_`
 #include "numkong/dot/serial.h" // `nk_u1x8_popcount_`
 
 #if defined(__cplusplus)
@@ -67,7 +68,7 @@ NK_PUBLIC void nk_dot_f16_svehalf(nk_f16_t const *a_scalars, nk_f16_t const *b_s
 
         idx_scalars += svcnth();
     } while (idx_scalars < count_scalars);
-    *result = svaddv_f32(svptrue_b32(), ab_f32x);
+    *result = nk_svaddv_f32_(svptrue_b32(), ab_f32x);
 }
 
 NK_PUBLIC void nk_dot_f16c_svehalf(nk_f16c_t const *a_pairs, nk_f16c_t const *b_pairs, nk_size_t count_pairs,
@@ -107,8 +108,8 @@ NK_PUBLIC void nk_dot_f16c_svehalf(nk_f16c_t const *a_pairs, nk_f16c_t const *b_
 
         idx_scalars += svcnth();
     } while (idx_scalars < count_pairs);
-    results->real = svaddv_f32(svptrue_b32(), ab_real_f32x);
-    results->imag = svaddv_f32(svptrue_b32(), ab_imag_f32x);
+    results->real = nk_svaddv_f32_(svptrue_b32(), ab_real_f32x);
+    results->imag = nk_svaddv_f32_(svptrue_b32(), ab_imag_f32x);
 }
 
 NK_PUBLIC void nk_vdot_f16c_svehalf(nk_f16c_t const *a_pairs, nk_f16c_t const *b_pairs, nk_size_t count_pairs,
@@ -148,8 +149,8 @@ NK_PUBLIC void nk_vdot_f16c_svehalf(nk_f16c_t const *a_pairs, nk_f16c_t const *b
 
         idx_scalars += svcnth();
     } while (idx_scalars < count_pairs);
-    results->real = svaddv_f32(svptrue_b32(), ab_real_f32x);
-    results->imag = svaddv_f32(svptrue_b32(), ab_imag_f32x);
+    results->real = nk_svaddv_f32_(svptrue_b32(), ab_real_f32x);
+    results->imag = nk_svaddv_f32_(svptrue_b32(), ab_imag_f32x);
 }
 
 #if defined(__clang__)

package/include/numkong/dot/svesdot.h

@@ -34,6 +34,7 @@
 #if NK_TARGET_SVESDOT
 
 #include "numkong/types.h"
+#include "numkong/reduce/sve.h" // `nk_svaddv_f64_`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -57,7 +58,7 @@ NK_PUBLIC void nk_dot_i8_svesdot(nk_i8_t const *a_scalars, nk_i8_t const *b_scal
         sum_i32x = svdot_s32(sum_i32x, a_i8x, b_i8x);
         idx_scalars += svcntb();
     } while (idx_scalars < count_scalars);
-    *result = (nk_i32_t)svaddv_s32(svptrue_b32(), sum_i32x);
+    *result = (nk_i32_t)nk_svaddv_s32_(svptrue_b32(), sum_i32x);
 }
 
 NK_PUBLIC void nk_dot_u8_svesdot(nk_u8_t const *a_scalars, nk_u8_t const *b_scalars, nk_size_t count_scalars,
@@ -71,7 +72,7 @@ NK_PUBLIC void nk_dot_u8_svesdot(nk_u8_t const *a_scalars, nk_u8_t const *b_scal
         sum_u32x = svdot_u32(sum_u32x, a_u8x, b_u8x);
         idx_scalars += svcntb();
     } while (idx_scalars < count_scalars);
-    *result = (nk_u32_t)svaddv_u32(svptrue_b32(), sum_u32x);
+    *result = (nk_u32_t)nk_svaddv_u32_(svptrue_b32(), sum_u32x);
 }
 
 #if defined(__clang__)