numkong 7.0.0 → 7.4.1

package/include/numkong/each/neon.h

@@ -8,20 +8,19 @@
  *
  *  @section elementwise_neon_instructions ARM NEON Instructions
  *
- *      Intrinsic         Instruction                   Latency     Throughput
- *                                                                  A76     M4+/V1+/Oryon
- *      vld1q_f32         LD1 (V.4S)                    4cy         2/cy    2/cy
- *      vst1q_f32         ST1 (V.4S)                    2cy         2/cy    2/cy
- *      vaddq_f32         FADD (V.4S, V.4S, V.4S)       2cy         2/cy    4/cy
- *      vmulq_f32         FMUL (V.4S, V.4S, V.4S)       3cy         2/cy    4/cy
- *      vfmaq_f32         FMLA (V.4S, V.4S, V.4S)       4cy         2/cy    4/cy
- *      vaddq_f64         FADD (V.2D, V.2D, V.2D)       2cy         2/cy    4/cy
- *      vmulq_f64         FMUL (V.2D, V.2D, V.2D)       3cy         2/cy    4/cy
- *      vfmaq_f64         FMLA (V.2D, V.2D, V.2D)       4cy         2/cy    4/cy
- *      vqaddq_s16        SQADD (V.8H, V.8H, V.8H)      2cy         2/cy    4/cy
- *      vcvtq_f32_s32     SCVTF (V.4S, V.4S)            3cy         2/cy    2/cy
- *      vcvtnq_s32_f32    FCVTNS (V.4S, V.4S)           3cy         2/cy    2/cy
- *      vqmovn_s32        SQXTN (V.4H, V.4S)            3cy         2/cy    2/cy
+ *      Intrinsic       Instruction               A76       M5
+ *      vld1q_f32       LD1 (V.4S)                4cy @ 2p  4cy @ 3p
+ *      vst1q_f32       ST1 (V.4S)                2cy @ 2p  2cy @ 3p
+ *      vaddq_f32       FADD (V.4S, V.4S, V.4S)   2cy @ 2p  2cy @ 4p
+ *      vmulq_f32       FMUL (V.4S, V.4S, V.4S)   3cy @ 2p  3cy @ 4p
+ *      vfmaq_f32       FMLA (V.4S, V.4S, V.4S)   4cy @ 2p  3cy @ 4p
+ *      vaddq_f64       FADD (V.2D, V.2D, V.2D)   2cy @ 2p  2cy @ 4p
+ *      vmulq_f64       FMUL (V.2D, V.2D, V.2D)   3cy @ 2p  3cy @ 4p
+ *      vfmaq_f64       FMLA (V.2D, V.2D, V.2D)   4cy @ 2p  3cy @ 4p
+ *      vqaddq_s16      SQADD (V.8H, V.8H, V.8H)  2cy @ 2p  3cy @ 2p
+ *      vcvtq_f32_s32   SCVTF (V.4S, V.4S)        3cy @ 2p  3cy @ 4p
+ *      vcvtnq_s32_f32  FCVTNS (V.4S, V.4S)       3cy @ 2p  3cy @ 4p
+ *      vqmovn_s32      SQXTN (V.4H, V.4S)        3cy @ 2p  3cy @ 4p
  *
  *  Elementwise operations are throughput-bound rather than latency-bound. FP arithmetic
  *  throughput doubles on 4-pipe cores (Apple M4+, Graviton3+, Oryon) from 2/cy to 4/cy.
@@ -37,6 +36,7 @@
 
 #include "numkong/types.h"
 #include "numkong/cast/neon.h"
+#include "numkong/cast/serial.h" // `nk_f32_to_u8_serial`, `nk_f32_to_i8_serial`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -145,10 +145,10 @@ NK_PUBLIC void nk_each_sum_i16_neon(nk_i16_t const *a, nk_i16_t const *b, nk_siz
     // The main loop:
     nk_size_t i = 0;
     for (; i + 8 <= n; i += 8) {
-        int16x8_t a_s16x8 = vld1q_s16(a + i);
-        int16x8_t b_s16x8 = vld1q_s16(b + i);
-        int16x8_t sum_s16x8 = vqaddq_s16(a_s16x8, b_s16x8);
-        vst1q_s16(result + i, sum_s16x8);
+        int16x8_t a_i16x8 = vld1q_s16(a + i);
+        int16x8_t b_i16x8 = vld1q_s16(b + i);
+        int16x8_t sum_i16x8 = vqaddq_s16(a_i16x8, b_i16x8);
+        vst1q_s16(result + i, sum_i16x8);
     }
 
     // The tail:
@@ -291,10 +291,10 @@ NK_PUBLIC void nk_each_sum_i32_neon(nk_i32_t const *a, nk_i32_t const *b, nk_siz
     // The main loop:
     nk_size_t i = 0;
     for (; i + 4 <= n; i += 4) {
-        int32x4_t a_s32x4 = vld1q_s32(a + i);
-        int32x4_t b_s32x4 = vld1q_s32(b + i);
-        int32x4_t sum_s32x4 = vqaddq_s32(a_s32x4, b_s32x4);
-        vst1q_s32(result + i, sum_s32x4);
+        int32x4_t a_i32x4 = vld1q_s32(a + i);
+        int32x4_t b_i32x4 = vld1q_s32(b + i);
+        int32x4_t sum_i32x4 = vqaddq_s32(a_i32x4, b_i32x4);
+        vst1q_s32(result + i, sum_i32x4);
     }
 
     // The tail:
@@ -437,10 +437,10 @@ NK_PUBLIC void nk_each_sum_i64_neon(nk_i64_t const *a, nk_i64_t const *b, nk_siz
     // The main loop:
     nk_size_t i = 0;
     for (; i + 2 <= n; i += 2) {
-        int64x2_t a_s64x2 = vld1q_s64(a + i);
-        int64x2_t b_s64x2 = vld1q_s64(b + i);
-        int64x2_t sum_s64x2 = vqaddq_s64(a_s64x2, b_s64x2);
-        vst1q_s64(result + i, sum_s64x2);
+        int64x2_t a_i64x2 = vld1q_s64(a + i);
+        int64x2_t b_i64x2 = vld1q_s64(b + i);
+        int64x2_t sum_i64x2 = vqaddq_s64(a_i64x2, b_i64x2);
+        vst1q_s64(result + i, sum_i64x2);
     }
 
     // The tail:
@@ -679,9 +679,9 @@ NK_PUBLIC void nk_each_sum_e4m3_neon(nk_e4m3_t const *a, nk_e4m3_t const *b, nk_
         float16x8_t a_f16x8 = nk_e4m3x8_to_f16x8_neon_(vld1_u8(a + i));
         float16x8_t b_f16x8 = nk_e4m3x8_to_f16x8_neon_(vld1_u8(b + i));
         float32x4_t a_low_f32x4 = vcvt_f32_f16(vget_low_f16(a_f16x8));
-        float32x4_t a_high_f32x4 = vcvt_f32_f16(vget_high_f16(a_f16x8));
+        float32x4_t a_high_f32x4 = vcvt_high_f32_f16(a_f16x8);
         float32x4_t b_low_f32x4 = vcvt_f32_f16(vget_low_f16(b_f16x8));
-        float32x4_t b_high_f32x4 = vcvt_f32_f16(vget_high_f16(b_f16x8));
+        float32x4_t b_high_f32x4 = vcvt_high_f32_f16(b_f16x8);
         float32x4_t result_low_f32x4 = vaddq_f32(a_low_f32x4, b_low_f32x4);
         float32x4_t result_high_f32x4 = vaddq_f32(a_high_f32x4, b_high_f32x4);
         nk_b32_vec_t result_low_vec = nk_f32x4_to_e4m3x4_neon_(result_low_f32x4);
@@ -703,9 +703,9 @@ NK_PUBLIC void nk_each_sum_e5m2_neon(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_
         float16x8_t a_f16x8 = nk_e5m2x8_to_f16x8_neon_(vld1_u8(a + i));
         float16x8_t b_f16x8 = nk_e5m2x8_to_f16x8_neon_(vld1_u8(b + i));
         float32x4_t a_low_f32x4 = vcvt_f32_f16(vget_low_f16(a_f16x8));
-        float32x4_t a_high_f32x4 = vcvt_f32_f16(vget_high_f16(a_f16x8));
+        float32x4_t a_high_f32x4 = vcvt_high_f32_f16(a_f16x8);
         float32x4_t b_low_f32x4 = vcvt_f32_f16(vget_low_f16(b_f16x8));
-        float32x4_t b_high_f32x4 = vcvt_f32_f16(vget_high_f16(b_f16x8));
+        float32x4_t b_high_f32x4 = vcvt_high_f32_f16(b_f16x8);
         float32x4_t result_low_f32x4 = vaddq_f32(a_low_f32x4, b_low_f32x4);
         float32x4_t result_high_f32x4 = vaddq_f32(a_high_f32x4, b_high_f32x4);
         nk_b32_vec_t result_low_vec = nk_f32x4_to_e5m2x4_neon_(result_low_f32x4);
@@ -729,7 +729,7 @@ NK_PUBLIC void nk_each_scale_e4m3_neon(nk_e4m3_t const *a, nk_size_t n, nk_f32_t
     for (; i + 8 <= n; i += 8) {
         float16x8_t a_f16x8 = nk_e4m3x8_to_f16x8_neon_(vld1_u8(a + i));
         float32x4_t a_low_f32x4 = vcvt_f32_f16(vget_low_f16(a_f16x8));
-        float32x4_t a_high_f32x4 = vcvt_f32_f16(vget_high_f16(a_f16x8));
+        float32x4_t a_high_f32x4 = vcvt_high_f32_f16(a_f16x8);
         float32x4_t result_low_f32x4 = vfmaq_f32(beta_f32x4, a_low_f32x4, alpha_f32x4);
         float32x4_t result_high_f32x4 = vfmaq_f32(beta_f32x4, a_high_f32x4, alpha_f32x4);
         nk_b32_vec_t result_low_vec = nk_f32x4_to_e4m3x4_neon_(result_low_f32x4);
@@ -752,7 +752,7 @@ NK_PUBLIC void nk_each_scale_e5m2_neon(nk_e5m2_t const *a, nk_size_t n, nk_f32_t
     for (; i + 8 <= n; i += 8) {
         float16x8_t a_f16x8 = nk_e5m2x8_to_f16x8_neon_(vld1_u8(a + i));
         float32x4_t a_low_f32x4 = vcvt_f32_f16(vget_low_f16(a_f16x8));
-        float32x4_t a_high_f32x4 = vcvt_f32_f16(vget_high_f16(a_f16x8));
+        float32x4_t a_high_f32x4 = vcvt_high_f32_f16(a_f16x8);
         float32x4_t result_low_f32x4 = vfmaq_f32(beta_f32x4, a_low_f32x4, alpha_f32x4);
         float32x4_t result_high_f32x4 = vfmaq_f32(beta_f32x4, a_high_f32x4, alpha_f32x4);
         nk_b32_vec_t result_low_vec = nk_f32x4_to_e5m2x4_neon_(result_low_f32x4);
@@ -776,9 +776,9 @@ NK_PUBLIC void nk_each_blend_e4m3_neon(nk_e4m3_t const *a, nk_e4m3_t const *b, n
         float16x8_t a_f16x8 = nk_e4m3x8_to_f16x8_neon_(vld1_u8(a + i));
         float16x8_t b_f16x8 = nk_e4m3x8_to_f16x8_neon_(vld1_u8(b + i));
         float32x4_t a_low_f32x4 = vcvt_f32_f16(vget_low_f16(a_f16x8));
-        float32x4_t a_high_f32x4 = vcvt_f32_f16(vget_high_f16(a_f16x8));
+        float32x4_t a_high_f32x4 = vcvt_high_f32_f16(a_f16x8);
         float32x4_t b_low_f32x4 = vcvt_f32_f16(vget_low_f16(b_f16x8));
-        float32x4_t b_high_f32x4 = vcvt_f32_f16(vget_high_f16(b_f16x8));
+        float32x4_t b_high_f32x4 = vcvt_high_f32_f16(b_f16x8);
         float32x4_t a_scaled_low_f32x4 = vmulq_f32(a_low_f32x4, alpha_f32x4);
         float32x4_t a_scaled_high_f32x4 = vmulq_f32(a_high_f32x4, alpha_f32x4);
         float32x4_t result_low_f32x4 = vfmaq_f32(a_scaled_low_f32x4, b_low_f32x4, beta_f32x4);
@@ -805,9 +805,9 @@ NK_PUBLIC void nk_each_blend_e5m2_neon(nk_e5m2_t const *a, nk_e5m2_t const *b, n
         float16x8_t a_f16x8 = nk_e5m2x8_to_f16x8_neon_(vld1_u8(a + i));
         float16x8_t b_f16x8 = nk_e5m2x8_to_f16x8_neon_(vld1_u8(b + i));
         float32x4_t a_low_f32x4 = vcvt_f32_f16(vget_low_f16(a_f16x8));
-        float32x4_t a_high_f32x4 = vcvt_f32_f16(vget_high_f16(a_f16x8));
+        float32x4_t a_high_f32x4 = vcvt_high_f32_f16(a_f16x8);
         float32x4_t b_low_f32x4 = vcvt_f32_f16(vget_low_f16(b_f16x8));
-        float32x4_t b_high_f32x4 = vcvt_f32_f16(vget_high_f16(b_f16x8));
+        float32x4_t b_high_f32x4 = vcvt_high_f32_f16(b_f16x8);
         float32x4_t a_scaled_low_f32x4 = vmulq_f32(a_low_f32x4, alpha_f32x4);
         float32x4_t a_scaled_high_f32x4 = vmulq_f32(a_high_f32x4, alpha_f32x4);
         float32x4_t result_low_f32x4 = vfmaq_f32(a_scaled_low_f32x4, b_low_f32x4, beta_f32x4);
@@ -835,11 +835,11 @@ NK_PUBLIC void nk_each_fma_e4m3_neon(nk_e4m3_t const *a, nk_e4m3_t const *b, nk_
         float16x8_t b_f16x8 = nk_e4m3x8_to_f16x8_neon_(vld1_u8(b + i));
         float16x8_t c_f16x8 = nk_e4m3x8_to_f16x8_neon_(vld1_u8(c + i));
         float32x4_t a_low_f32x4 = vcvt_f32_f16(vget_low_f16(a_f16x8));
-        float32x4_t a_high_f32x4 = vcvt_f32_f16(vget_high_f16(a_f16x8));
+        float32x4_t a_high_f32x4 = vcvt_high_f32_f16(a_f16x8);
         float32x4_t b_low_f32x4 = vcvt_f32_f16(vget_low_f16(b_f16x8));
-        float32x4_t b_high_f32x4 = vcvt_f32_f16(vget_high_f16(b_f16x8));
+        float32x4_t b_high_f32x4 = vcvt_high_f32_f16(b_f16x8);
         float32x4_t c_low_f32x4 = vcvt_f32_f16(vget_low_f16(c_f16x8));
-        float32x4_t c_high_f32x4 = vcvt_f32_f16(vget_high_f16(c_f16x8));
+        float32x4_t c_high_f32x4 = vcvt_high_f32_f16(c_f16x8);
         float32x4_t ab_low_f32x4 = vmulq_f32(a_low_f32x4, b_low_f32x4);
         float32x4_t ab_high_f32x4 = vmulq_f32(a_high_f32x4, b_high_f32x4);
         float32x4_t ab_scaled_low_f32x4 = vmulq_f32(ab_low_f32x4, alpha_f32x4);
@@ -870,11 +870,11 @@ NK_PUBLIC void nk_each_fma_e5m2_neon(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_
         float16x8_t b_f16x8 = nk_e5m2x8_to_f16x8_neon_(vld1_u8(b + i));
         float16x8_t c_f16x8 = nk_e5m2x8_to_f16x8_neon_(vld1_u8(c + i));
         float32x4_t a_low_f32x4 = vcvt_f32_f16(vget_low_f16(a_f16x8));
-        float32x4_t a_high_f32x4 = vcvt_f32_f16(vget_high_f16(a_f16x8));
+        float32x4_t a_high_f32x4 = vcvt_high_f32_f16(a_f16x8);
         float32x4_t b_low_f32x4 = vcvt_f32_f16(vget_low_f16(b_f16x8));
-        float32x4_t b_high_f32x4 = vcvt_f32_f16(vget_high_f16(b_f16x8));
+        float32x4_t b_high_f32x4 = vcvt_high_f32_f16(b_f16x8);
         float32x4_t c_low_f32x4 = vcvt_f32_f16(vget_low_f16(c_f16x8));
-        float32x4_t c_high_f32x4 = vcvt_f32_f16(vget_high_f16(c_f16x8));
+        float32x4_t c_high_f32x4 = vcvt_high_f32_f16(c_f16x8);
         float32x4_t ab_low_f32x4 = vmulq_f32(a_low_f32x4, b_low_f32x4);
         float32x4_t ab_high_f32x4 = vmulq_f32(a_high_f32x4, b_high_f32x4);
         float32x4_t ab_scaled_low_f32x4 = vmulq_f32(ab_low_f32x4, alpha_f32x4);
@@ -1089,6 +1089,40 @@ NK_PUBLIC void nk_each_fma_f64c_neon(nk_f64c_t const *a, nk_f64c_t const *b, nk_
     }
 }
 
+NK_PUBLIC void nk_each_sum_u8_neon(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_u8_t *result) {
+    // The main loop:
+    nk_size_t i = 0;
+    for (; i + 16 <= n; i += 16) {
+        uint8x16_t a_vec = vld1q_u8(a + i);
+        uint8x16_t b_vec = vld1q_u8(b + i);
+        uint8x16_t sum_vec = vqaddq_u8(a_vec, b_vec);
+        vst1q_u8(result + i, sum_vec);
+    }
+
+    // The tail:
+    for (; i < n; ++i) {
+        nk_f32_t sum = (nk_f32_t)a[i] + b[i];
+        nk_f32_to_u8_serial(&sum, result + i);
+    }
+}
+
+NK_PUBLIC void nk_each_sum_i8_neon(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_i8_t *result) {
+    // The main loop:
+    nk_size_t i = 0;
+    for (; i + 16 <= n; i += 16) {
+        int8x16_t a_vec = vld1q_s8(a + i);
+        int8x16_t b_vec = vld1q_s8(b + i);
+        int8x16_t sum_vec = vqaddq_s8(a_vec, b_vec);
+        vst1q_s8(result + i, sum_vec);
+    }
+
+    // The tail:
+    for (; i < n; ++i) {
+        nk_f32_t sum = (nk_f32_t)a[i] + b[i];
+        nk_f32_to_i8_serial(&sum, result + i);
+    }
+}
+
 #if defined(__clang__)
 #pragma clang attribute pop
 #elif defined(__GNUC__)

package/include/numkong/each/neonbfdot.h

@@ -8,18 +8,17 @@
  *
  *  @section elementwise_neonbfdot_instructions ARM NEON BF16 Instructions (ARMv8.6-BF16)
  *
- *      Intrinsic                   Instruction                     Latency     Throughput
- *                                                                              A76         M4+/V1+/Oryon
- *      vld1_bf16                   LD1 (V.4H)                      4cy         2/cy        3/cy
- *      vst1_bf16                   ST1 (V.4H)                      2cy         2/cy        3/cy
- *      vcvt_f32_bf16               BFCVTN (V.4H, V.4S)             3cy         2/cy        4/cy
- *      vcvt_bf16_f32               BFCVT (V.4H, V.4S)              3cy         2/cy        4/cy
- *      vaddq_f32                   FADD (V.4S, V.4S, V.4S)         2cy         2/cy        4/cy
- *      vmulq_f32                   FMUL (V.4S, V.4S, V.4S)         3cy         2/cy        4/cy
- *      vmulq_n_f32                 FMUL (V.4S, V.4S, scalar)       3cy         2/cy        4/cy
- *      vfmaq_f32                   FMLA (V.4S, V.4S, V.4S)         4cy         2/cy        4/cy
- *      vfmaq_n_f32                 FMLA (V.4S, V.4S, scalar)       4cy         2/cy        4/cy
- *      vdupq_n_f32                 DUP (V.4S, scalar)              2cy         2/cy        4/cy
+ *      Intrinsic      Instruction                A76       M5
+ *      vld1_bf16      LD1 (V.4H)                 4cy @ 2p  4cy @ 3p
+ *      vst1_bf16      ST1 (V.4H)                 2cy @ 2p  2cy @ 3p
+ *      vcvt_f32_bf16  BFCVTN (V.4H, V.4S)        3cy @ 2p  3cy @ 4p
+ *      vcvt_bf16_f32  BFCVT (V.4H, V.4S)         3cy @ 2p  3cy @ 4p
+ *      vaddq_f32      FADD (V.4S, V.4S, V.4S)    2cy @ 2p  2cy @ 4p
+ *      vmulq_f32      FMUL (V.4S, V.4S, V.4S)    3cy @ 2p  3cy @ 4p
+ *      vmulq_n_f32    FMUL (V.4S, V.4S, scalar)  3cy @ 2p  3cy @ 4p
+ *      vfmaq_f32      FMLA (V.4S, V.4S, V.4S)    4cy @ 2p  3cy @ 4p
+ *      vfmaq_n_f32    FMLA (V.4S, V.4S, scalar)  4cy @ 2p  3cy @ 4p
+ *      vdupq_n_f32    DUP (V.4S, scalar)         2cy @ 2p  2cy @ 4p
  *
  *  The ARMv8.6-BF16 extension provides element-wise operations on BF16 data by converting to F32
  *  for arithmetic, then back to BF16 for storage. This preserves the dynamic range benefits of BF16

package/include/numkong/each/neonhalf.h

@@ -8,28 +8,27 @@
  *
  *  @section elementwise_neonhalf_instructions ARM NEON FP16 Instructions (ARMv8.2-FP16)
  *
- *      Intrinsic                   Instruction                     Latency     Throughput
- *                                                                              A76         M4+/V1+/Oryon
- *      vld1q_f16                   LD1 (V.8H)                      4cy         2/cy        3/cy
- *      vst1q_f16                   ST1 (V.8H)                      2cy         2/cy        3/cy
- *      vaddq_f16                   FADD (V.8H, V.8H, V.8H)         2cy         2/cy        4/cy
- *      vmulq_f16                   FMUL (V.8H, V.8H, V.8H)         3cy         2/cy        4/cy
- *      vmulq_n_f16                 FMUL (V.8H, V.8H, scalar)       3cy         2/cy        4/cy
- *      vfmaq_f16                   FMLA (V.8H, V.8H, V.8H)         4cy         2/cy        4/cy
- *      vfmaq_n_f16                 FMLA (V.8H, V.8H, scalar)       4cy         2/cy        4/cy
- *      vdupq_n_f16                 DUP (V.8H, scalar)              2cy         2/cy        4/cy
- *      vld1_u8                     LD1 (V.8B)                      4cy         2/cy        3/cy
- *      vld1_s8                     LD1 (V.8B)                      4cy         2/cy        3/cy
- *      vmovl_u8                    UXTL (V.8H, V.8B)               2cy         2/cy        4/cy
- *      vmovl_s8                    SXTL (V.8H, V.8B)               2cy         2/cy        4/cy
- *      vcvtq_f16_u16               UCVTF (V.8H, V.8H)              3cy         2/cy        4/cy
- *      vcvtq_f16_s16               SCVTF (V.8H, V.8H)              3cy         2/cy        4/cy
- *      vcvtnq_u16_f16              FCVTNU (V.8H, V.8H)             3cy         2/cy        4/cy
- *      vcvtnq_s16_f16              FCVTNS (V.8H, V.8H)             3cy         2/cy        4/cy
- *      vqmovn_u16                  UQXTN (V.8B, V.8H)              3cy         2/cy        4/cy
- *      vqmovn_s16                  SQXTN (V.8B, V.8H)              3cy         2/cy        4/cy
- *      vqaddq_u8                   UQADD (V.16B, V.16B, V.16B)     2cy         2/cy        4/cy
- *      vqaddq_s8                   SQADD (V.16B, V.16B, V.16B)     2cy         2/cy        4/cy
+ *      Intrinsic       Instruction                  A76       M5
+ *      vld1q_f16       LD1 (V.8H)                   4cy @ 2p  4cy @ 3p
+ *      vst1q_f16       ST1 (V.8H)                   2cy @ 2p  2cy @ 3p
+ *      vaddq_f16       FADD (V.8H, V.8H, V.8H)      2cy @ 2p  2cy @ 4p
+ *      vmulq_f16       FMUL (V.8H, V.8H, V.8H)      3cy @ 2p  3cy @ 4p
+ *      vmulq_n_f16     FMUL (V.8H, V.8H, scalar)    3cy @ 2p  3cy @ 4p
+ *      vfmaq_f16       FMLA (V.8H, V.8H, V.8H)      4cy @ 2p  4cy @ 4p
+ *      vfmaq_n_f16     FMLA (V.8H, V.8H, scalar)    4cy @ 2p  4cy @ 4p
+ *      vdupq_n_f16     DUP (V.8H, scalar)           2cy @ 2p  2cy @ 4p
+ *      vld1_u8         LD1 (V.8B)                   4cy @ 2p  4cy @ 3p
+ *      vld1_s8         LD1 (V.8B)                   4cy @ 2p  4cy @ 3p
+ *      vmovl_u8        UXTL (V.8H, V.8B)            2cy @ 2p  2cy @ 4p
+ *      vmovl_s8        SXTL (V.8H, V.8B)            2cy @ 2p  2cy @ 4p
+ *      vcvtq_f16_u16   UCVTF (V.8H, V.8H)           3cy @ 2p  3cy @ 4p
+ *      vcvtq_f16_s16   SCVTF (V.8H, V.8H)           3cy @ 2p  3cy @ 4p
+ *      vcvtnq_u16_f16  FCVTNU (V.8H, V.8H)          3cy @ 2p  3cy @ 4p
+ *      vcvtnq_s16_f16  FCVTNS (V.8H, V.8H)          3cy @ 2p  3cy @ 4p
+ *      vqmovn_u16      UQXTN (V.8B, V.8H)           3cy @ 2p  3cy @ 4p
+ *      vqmovn_s16      SQXTN (V.8B, V.8H)           3cy @ 2p  3cy @ 4p
+ *      vqaddq_u8       UQADD (V.16B, V.16B, V.16B)  2cy @ 2p  3cy @ 2p
+ *      vqaddq_s8       SQADD (V.16B, V.16B, V.16B)  2cy @ 2p  3cy @ 2p
  *
  *  The ARMv8.2-FP16 extension enables native half-precision element-wise operations, processing 8
  *  F16 elements per instruction. Operations like sum, scale, blend, and fma work directly in F16,
@@ -46,6 +45,7 @@
 
 #include "numkong/types.h"
 #include "numkong/cast/serial.h" // `nk_f32_to_i8_serial`
+#include "numkong/each/neon.h"   // `nk_each_sum_u8_neon`, `nk_each_sum_i8_neon`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -161,23 +161,6 @@ NK_PUBLIC void nk_each_fma_f16_neonhalf(                     //
                                    beta_f16 * ((float16_t const *)c)[i];
 }
 
-NK_PUBLIC void nk_each_sum_u8_neonhalf(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_u8_t *result) {
-    // The main loop:
-    nk_size_t i = 0;
-    for (; i + 16 <= n; i += 16) {
-        uint8x16_t a_vec = vld1q_u8(a + i);
-        uint8x16_t b_vec = vld1q_u8(b + i);
-        uint8x16_t sum_vec = vqaddq_u8(a_vec, b_vec);
-        vst1q_u8(result + i, sum_vec);
-    }
-
-    // The tail:
-    for (; i < n; ++i) {
-        nk_f32_t sum = (nk_f32_t)a[i] + b[i];
-        nk_f32_to_u8_serial(&sum, result + i);
-    }
-}
-
 NK_PUBLIC void nk_each_scale_u8_neonhalf(nk_u8_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
                                          nk_u8_t *result) {
     float16_t alpha_f16 = (float16_t)*alpha;
@@ -213,7 +196,7 @@ NK_PUBLIC void nk_each_blend_u8_neonhalf(            //
     // 1. Simple addition, when both weights are equal to 1.0.
     if (alpha_val == 1 && beta_val == 1) {
         // In this case we can avoid expensive multiplications.
-        nk_each_sum_u8_neonhalf(a, b, n, result);
+        nk_each_sum_u8_neon(a, b, n, result);
         return;
     }
     // 2. Just scaling, when one of the weights is equal to zero.
@@ -249,52 +232,6 @@ NK_PUBLIC void nk_each_blend_u8_neonhalf(            //
     }
 }
 
-NK_PUBLIC void nk_each_fma_u8_neonhalf(                   //
-    nk_u8_t const *a, nk_u8_t const *b, nk_u8_t const *c, //
-    nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta, nk_u8_t *result) {
-    float16_t alpha_f16 = (float16_t)*alpha;
-    float16_t beta_f16 = (float16_t)*beta;
-
-    // The main loop:
-    nk_size_t i = 0;
-    for (; i + 8 <= n; i += 8) {
-        uint8x8_t a_u8x8 = vld1_u8(a + i);
-        uint8x8_t b_u8x8 = vld1_u8(b + i);
-        uint8x8_t c_u8x8 = vld1_u8(c + i);
-        float16x8_t a_f16x8 = vcvtq_f16_u16(vmovl_u8(a_u8x8));
-        float16x8_t b_f16x8 = vcvtq_f16_u16(vmovl_u8(b_u8x8));
-        float16x8_t c_f16x8 = vcvtq_f16_u16(vmovl_u8(c_u8x8));
-        float16x8_t ab_f16x8 = vmulq_f16(a_f16x8, b_f16x8);
-        float16x8_t ab_scaled_f16x8 = vmulq_n_f16(ab_f16x8, alpha_f16);
-        float16x8_t result_f16x8 = vfmaq_n_f16(ab_scaled_f16x8, c_f16x8, beta_f16);
-        uint8x8_t result_u8x8 = vqmovn_u16(vcvtnq_u16_f16(result_f16x8));
-        vst1_u8(result + i, result_u8x8);
-    }
-
-    // The tail:
-    for (; i < n; ++i) {
-        nk_f32_t sum = alpha_f16 * a[i] * b[i] + beta_f16 * c[i];
-        nk_f32_to_u8_serial(&sum, result + i);
-    }
-}
-
-NK_PUBLIC void nk_each_sum_i8_neonhalf(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_i8_t *result) {
-    // The main loop:
-    nk_size_t i = 0;
-    for (; i + 16 <= n; i += 16) {
-        int8x16_t a_vec = vld1q_s8(a + i);
-        int8x16_t b_vec = vld1q_s8(b + i);
-        int8x16_t sum_vec = vqaddq_s8(a_vec, b_vec);
-        vst1q_s8(result + i, sum_vec);
-    }
-
-    // The tail:
-    for (; i < n; ++i) {
-        nk_f32_t sum = (nk_f32_t)a[i] + b[i];
-        nk_f32_to_i8_serial(&sum, result + i);
-    }
-}
-
 NK_PUBLIC void nk_each_scale_i8_neonhalf(nk_i8_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
                                          nk_i8_t *result) {
     float16_t alpha_f16 = (float16_t)*alpha;
@@ -330,7 +267,7 @@ NK_PUBLIC void nk_each_blend_i8_neonhalf(            //
     // 1. Simple addition, when both weights are equal to 1.0.
     if (alpha_val == 1 && beta_val == 1) {
         // In this case we can avoid expensive multiplications.
-        nk_each_sum_i8_neonhalf(a, b, n, result);
+        nk_each_sum_i8_neon(a, b, n, result);
         return;
     }
     // 2. Just scaling, when one of the weights is equal to zero.
@@ -366,35 +303,6 @@ NK_PUBLIC void nk_each_blend_i8_neonhalf(            //
     }
 }
 
-NK_PUBLIC void nk_each_fma_i8_neonhalf(                   //
-    nk_i8_t const *a, nk_i8_t const *b, nk_i8_t const *c, //
-    nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta, nk_i8_t *result) {
-    float16_t alpha_f16 = (float16_t)*alpha;
-    float16_t beta_f16 = (float16_t)*beta;
-
-    // The main loop:
-    nk_size_t i = 0;
-    for (; i + 8 <= n; i += 8) {
-        int8x8_t a_i8x8 = vld1_s8(a + i);
-        int8x8_t b_i8x8 = vld1_s8(b + i);
-        int8x8_t c_i8x8 = vld1_s8(c + i);
-        float16x8_t a_f16x8 = vcvtq_f16_s16(vmovl_s8(a_i8x8));
-        float16x8_t b_f16x8 = vcvtq_f16_s16(vmovl_s8(b_i8x8));
-        float16x8_t c_f16x8 = vcvtq_f16_s16(vmovl_s8(c_i8x8));
-        float16x8_t ab_f16x8 = vmulq_f16(a_f16x8, b_f16x8);
-        float16x8_t ab_scaled_f16x8 = vmulq_n_f16(ab_f16x8, alpha_f16);
-        float16x8_t result_f16x8 = vfmaq_n_f16(ab_scaled_f16x8, c_f16x8, beta_f16);
-        int8x8_t result_i8x8 = vqmovn_s16(vcvtnq_s16_f16(result_f16x8));
-        vst1_s8(result + i, result_i8x8);
-    }
-
-    // The tail:
-    for (; i < n; ++i) {
-        nk_f32_t sum = alpha_f16 * a[i] * b[i] + beta_f16 * c[i];
-        nk_f32_to_i8_serial(&sum, result + i);
-    }
-}
-
 #if defined(__clang__)
 #pragma clang attribute pop
 #elif defined(__GNUC__)

package/include/numkong/each/rvv.h

@@ -185,8 +185,8 @@ NK_PUBLIC void nk_each_sum_e5m2_rvv(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_s
 NK_PUBLIC void nk_each_scale_f64_rvv(nk_f64_t const *a, nk_size_t n, nk_f64_t const *alpha, nk_f64_t const *beta,
                                      nk_f64_t *result) {
     nk_f64_t alpha_val = *alpha, beta_val = *beta;
-    nk_size_t vlmax = __riscv_vsetvlmax_e64m4();
-    vfloat64m4_t beta_f64m4 = __riscv_vfmv_v_f_f64m4(beta_val, vlmax);
+    nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
+    vfloat64m4_t beta_f64m4 = __riscv_vfmv_v_f_f64m4(beta_val, max_vector_length);
     for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
         vector_length = __riscv_vsetvl_e64m4(n);
         vfloat64m4_t a_f64m4 = __riscv_vle64_v_f64m4(a, vector_length);
@@ -198,8 +198,8 @@ NK_PUBLIC void nk_each_scale_f64_rvv(nk_f64_t const *a, nk_size_t n, nk_f64_t co
 NK_PUBLIC void nk_each_scale_f32_rvv(nk_f32_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
                                      nk_f32_t *result) {
     nk_f32_t alpha_val = *alpha, beta_val = *beta;
-    nk_size_t vlmax = __riscv_vsetvlmax_e32m4();
-    vfloat32m4_t beta_f32m4 = __riscv_vfmv_v_f_f32m4(beta_val, vlmax);
+    nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
+    vfloat32m4_t beta_f32m4 = __riscv_vfmv_v_f_f32m4(beta_val, max_vector_length);
     for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
         vector_length = __riscv_vsetvl_e32m4(n);
         vfloat32m4_t a_f32m4 = __riscv_vle32_v_f32m4(a, vector_length);
@@ -211,8 +211,8 @@ NK_PUBLIC void nk_each_scale_f32_rvv(nk_f32_t const *a, nk_size_t n, nk_f32_t co
 NK_PUBLIC void nk_each_scale_f16_rvv(nk_f16_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
                                      nk_f16_t *result) {
     nk_f32_t alpha_val = *alpha, beta_val = *beta;
-    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
-    vfloat32m2_t beta_f32m2 = __riscv_vfmv_v_f_f32m2(beta_val, vlmax);
+    nk_size_t max_vector_length = __riscv_vsetvlmax_e32m2();
+    vfloat32m2_t beta_f32m2 = __riscv_vfmv_v_f_f32m2(beta_val, max_vector_length);
     for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
         vector_length = __riscv_vsetvl_e16m1(n);
         vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)a, vector_length);
@@ -226,8 +226,8 @@ NK_PUBLIC void nk_each_scale_f16_rvv(nk_f16_t const *a, nk_size_t n, nk_f32_t co
 NK_PUBLIC void nk_each_scale_bf16_rvv(nk_bf16_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
                                       nk_bf16_t *result) {
     nk_f32_t alpha_val = *alpha, beta_val = *beta;
-    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
-    vfloat32m2_t beta_f32m2 = __riscv_vfmv_v_f_f32m2(beta_val, vlmax);
+    nk_size_t max_vector_length = __riscv_vsetvlmax_e32m2();
+    vfloat32m2_t beta_f32m2 = __riscv_vfmv_v_f_f32m2(beta_val, max_vector_length);
     for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
         vector_length = __riscv_vsetvl_e16m1(n);
         vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)a, vector_length);
@@ -241,8 +241,8 @@ NK_PUBLIC void nk_each_scale_bf16_rvv(nk_bf16_t const *a, nk_size_t n, nk_f32_t
 NK_PUBLIC void nk_each_scale_i8_rvv(nk_i8_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
                                     nk_i8_t *result) {
     nk_f32_t alpha_val = *alpha, beta_val = *beta;
-    nk_size_t vlmax = __riscv_vsetvlmax_e32m4();
-    vfloat32m4_t beta_f32m4 = __riscv_vfmv_v_f_f32m4(beta_val, vlmax);
+    nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
+    vfloat32m4_t beta_f32m4 = __riscv_vfmv_v_f_f32m4(beta_val, max_vector_length);
     for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
         vector_length = __riscv_vsetvl_e8m1(n);
         vint8m1_t a_i8m1 = __riscv_vle8_v_i8m1(a, vector_length);
@@ -262,8 +262,8 @@ NK_PUBLIC void nk_each_scale_i8_rvv(nk_i8_t const *a, nk_size_t n, nk_f32_t cons
 NK_PUBLIC void nk_each_scale_u8_rvv(nk_u8_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
                                     nk_u8_t *result) {
     nk_f32_t alpha_val = *alpha, beta_val = *beta;
-    nk_size_t vlmax = __riscv_vsetvlmax_e32m4();
-    vfloat32m4_t beta_f32m4 = __riscv_vfmv_v_f_f32m4(beta_val, vlmax);
+    nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
+    vfloat32m4_t beta_f32m4 = __riscv_vfmv_v_f_f32m4(beta_val, max_vector_length);
     for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
         vector_length = __riscv_vsetvl_e8m1(n);
         vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1(a, vector_length);
@@ -283,8 +283,8 @@ NK_PUBLIC void nk_each_scale_u8_rvv(nk_u8_t const *a, nk_size_t n, nk_f32_t cons
 NK_PUBLIC void nk_each_scale_i16_rvv(nk_i16_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
                                      nk_i16_t *result) {
     nk_f32_t alpha_val = *alpha, beta_val = *beta;
-    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
-    vfloat32m2_t beta_f32m2 = __riscv_vfmv_v_f_f32m2(beta_val, vlmax);
+    nk_size_t max_vector_length = __riscv_vsetvlmax_e32m2();
+    vfloat32m2_t beta_f32m2 = __riscv_vfmv_v_f_f32m2(beta_val, max_vector_length);
     for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
         vector_length = __riscv_vsetvl_e16m1(n);
         vint16m1_t a_i16m1 = __riscv_vle16_v_i16m1(a, vector_length);
@@ -302,8 +302,8 @@ NK_PUBLIC void nk_each_scale_i16_rvv(nk_i16_t const *a, nk_size_t n, nk_f32_t co
 NK_PUBLIC void nk_each_scale_u16_rvv(nk_u16_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
                                      nk_u16_t *result) {
     nk_f32_t alpha_val = *alpha, beta_val = *beta;
-    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
-    vfloat32m2_t beta_f32m2 = __riscv_vfmv_v_f_f32m2(beta_val, vlmax);
+    nk_size_t max_vector_length = __riscv_vsetvlmax_e32m2();
+    vfloat32m2_t beta_f32m2 = __riscv_vfmv_v_f_f32m2(beta_val, max_vector_length);
     for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
         vector_length = __riscv_vsetvl_e16m1(n);
         vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1(a, vector_length);
@@ -321,8 +321,8 @@ NK_PUBLIC void nk_each_scale_u16_rvv(nk_u16_t const *a, nk_size_t n, nk_f32_t co
 NK_PUBLIC void nk_each_scale_i32_rvv(nk_i32_t const *a, nk_size_t n, nk_f64_t const *alpha, nk_f64_t const *beta,
                                      nk_i32_t *result) {
     nk_f64_t alpha_val = *alpha, beta_val = *beta;
-    nk_size_t vlmax = __riscv_vsetvlmax_e64m2();
-    vfloat64m2_t beta_f64m2 = __riscv_vfmv_v_f_f64m2(beta_val, vlmax);
+    nk_size_t max_vector_length = __riscv_vsetvlmax_e64m2();
+    vfloat64m2_t beta_f64m2 = __riscv_vfmv_v_f_f64m2(beta_val, max_vector_length);
     for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
         vector_length = __riscv_vsetvl_e32m1(n);
         vint32m1_t a_i32m1 = __riscv_vle32_v_i32m1(a, vector_length);
@@ -338,8 +338,8 @@ NK_PUBLIC void nk_each_scale_i32_rvv(nk_i32_t const *a, nk_size_t n, nk_f64_t co
 NK_PUBLIC void nk_each_scale_u32_rvv(nk_u32_t const *a, nk_size_t n, nk_f64_t const *alpha, nk_f64_t const *beta,
                                      nk_u32_t *result) {
     nk_f64_t alpha_val = *alpha, beta_val = *beta;
-    nk_size_t vlmax = __riscv_vsetvlmax_e64m2();
-    vfloat64m2_t beta_f64m2 = __riscv_vfmv_v_f_f64m2(beta_val, vlmax);
+    nk_size_t max_vector_length = __riscv_vsetvlmax_e64m2();
+    vfloat64m2_t beta_f64m2 = __riscv_vfmv_v_f_f64m2(beta_val, max_vector_length);
     for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
         vector_length = __riscv_vsetvl_e32m1(n);
         vuint32m1_t a_u32m1 = __riscv_vle32_v_u32m1(a, vector_length);
@@ -355,8 +355,8 @@ NK_PUBLIC void nk_each_scale_u32_rvv(nk_u32_t const *a, nk_size_t n, nk_f64_t co
 NK_PUBLIC void nk_each_scale_i64_rvv(nk_i64_t const *a, nk_size_t n, nk_f64_t const *alpha, nk_f64_t const *beta,
                                      nk_i64_t *result) {
     nk_f64_t alpha_val = *alpha, beta_val = *beta;
-    nk_size_t vlmax = __riscv_vsetvlmax_e64m4();
-    vfloat64m4_t beta_f64m4 = __riscv_vfmv_v_f_f64m4(beta_val, vlmax);
+    nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
+    vfloat64m4_t beta_f64m4 = __riscv_vfmv_v_f_f64m4(beta_val, max_vector_length);
     for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
         vector_length = __riscv_vsetvl_e64m4(n);
         vint64m4_t a_i64m4 = __riscv_vle64_v_i64m4(a, vector_length);
@@ -370,8 +370,8 @@ NK_PUBLIC void nk_each_scale_i64_rvv(nk_i64_t const *a, nk_size_t n, nk_f64_t co
 NK_PUBLIC void nk_each_scale_u64_rvv(nk_u64_t const *a, nk_size_t n, nk_f64_t const *alpha, nk_f64_t const *beta,
                                      nk_u64_t *result) {
     nk_f64_t alpha_val = *alpha, beta_val = *beta;
-    nk_size_t vlmax = __riscv_vsetvlmax_e64m4();
-    vfloat64m4_t beta_f64m4 = __riscv_vfmv_v_f_f64m4(beta_val, vlmax);
+    nk_size_t max_vector_length = __riscv_vsetvlmax_e64m4();
+    vfloat64m4_t beta_f64m4 = __riscv_vfmv_v_f_f64m4(beta_val, max_vector_length);
     for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
         vector_length = __riscv_vsetvl_e64m4(n);
         vuint64m4_t a_u64m4 = __riscv_vle64_v_u64m4(a, vector_length);
@@ -386,8 +386,8 @@ NK_PUBLIC void nk_each_scale_u64_rvv(nk_u64_t const *a, nk_size_t n, nk_f64_t co
 NK_PUBLIC void nk_each_scale_e4m3_rvv(nk_e4m3_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
                                       nk_e4m3_t *result) {
     nk_f32_t alpha_val = *alpha, beta_val = *beta;
-    nk_size_t vlmax = __riscv_vsetvlmax_e32m4();
-    vfloat32m4_t beta_f32m4 = __riscv_vfmv_v_f_f32m4(beta_val, vlmax);
+    nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
+    vfloat32m4_t beta_f32m4 = __riscv_vfmv_v_f_f32m4(beta_val, max_vector_length);
     for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
         vector_length = __riscv_vsetvl_e8m1(n);
         vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)a, vector_length);
@@ -401,8 +401,8 @@ NK_PUBLIC void nk_each_scale_e4m3_rvv(nk_e4m3_t const *a, nk_size_t n, nk_f32_t
 NK_PUBLIC void nk_each_scale_e5m2_rvv(nk_e5m2_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
                                       nk_e5m2_t *result) {
     nk_f32_t alpha_val = *alpha, beta_val = *beta;
-    nk_size_t vlmax = __riscv_vsetvlmax_e32m4();
-    vfloat32m4_t beta_f32m4 = __riscv_vfmv_v_f_f32m4(beta_val, vlmax);
+    nk_size_t max_vector_length = __riscv_vsetvlmax_e32m4();
+    vfloat32m4_t beta_f32m4 = __riscv_vfmv_v_f_f32m4(beta_val, max_vector_length);
     for (nk_size_t vector_length; n > 0; n -= vector_length, a += vector_length, result += vector_length) {
         vector_length = __riscv_vsetvl_e8m1(n);
         vuint8m1_t a_u8m1 = __riscv_vle8_v_u8m1((nk_u8_t const *)a, vector_length);