numkong 7.0.0 → 7.4.2

package/include/numkong/spatial/skylake.h

@@ -8,12 +8,11 @@
  *
  *  @section spatial_skylake_instructions Key AVX-512 Spatial Instructions
  *
- *      Intrinsic                   Instruction                     Latency     Throughput  Ports
- *      _mm512_fmadd_ps             VFMADD132PS (ZMM, ZMM, ZMM)     4cy         0.5/cy      p05
- *      _mm512_sub_ps               VSUBPS (ZMM, ZMM, ZMM)          4cy         0.5/cy      p05
- *      _mm512_rsqrt14_ps           VRSQRT14PS (ZMM, ZMM)           4cy         1/cy        p0
- *      _mm512_sqrt_ps              VSQRTPS (ZMM, ZMM)              12cy        3cy         p0
- *      _mm512_reduce_add_ps        (sequence)                      ~8-10cy     -           -
+ *      Intrinsic          Instruction                  Skylake-X         Genoa
+ *      _mm512_fmadd_ps    VFMADD132PS (ZMM, ZMM, ZMM)  4cy @ p05         4cy @ p01
+ *      _mm512_sub_ps      VSUBPS (ZMM, ZMM, ZMM)       4cy @ p05         3cy @ p23
+ *      _mm512_rsqrt14_ps  VRSQRT14PS (ZMM, ZMM)        7cy @ p0+p0+p05   5cy @ p01
+ *      _mm512_sqrt_ps     VSQRTPS (ZMM, ZMM)           20cy @ p0+p0+p05  15cy @ p01
  *
  *  Distance computations benefit from Skylake-X's dual FMA units achieving 0.5cy throughput for
  *  fused multiply-add operations. VRSQRT14PS provides ~14-bit precision reciprocal square root;
@@ -43,21 +42,21 @@ extern "C" {
 
 /** @brief Reciprocal square root of 16 floats with Newton-Raphson refinement (~28-bit precision). */
 NK_INTERNAL __m512 nk_rsqrt_f32x16_skylake_(__m512 x) {
-    __m512 rsqrt = _mm512_rsqrt14_ps(x);
-    __m512 nr = _mm512_mul_ps(_mm512_mul_ps(x, rsqrt), rsqrt);
-    nr = _mm512_sub_ps(_mm512_set1_ps(3.0f), nr);
-    return _mm512_mul_ps(_mm512_mul_ps(_mm512_set1_ps(0.5f), rsqrt), nr);
+    __m512 rsqrt_f32x16 = _mm512_rsqrt14_ps(x);
+    __m512 nr_f32x16 = _mm512_mul_ps(_mm512_mul_ps(x, rsqrt_f32x16), rsqrt_f32x16);
+    nr_f32x16 = _mm512_sub_ps(_mm512_set1_ps(3.0f), nr_f32x16);
+    return _mm512_mul_ps(_mm512_mul_ps(_mm512_set1_ps(0.5f), rsqrt_f32x16), nr_f32x16);
 }
 
 /** @brief Reciprocal square root of 8 doubles with Newton-Raphson refinement (~28-bit precision). */
 NK_INTERNAL __m512d nk_rsqrt_f64x8_skylake_(__m512d x) {
-    __m512d rsqrt = _mm512_rsqrt14_pd(x);
-    __m512d nr = _mm512_mul_pd(_mm512_mul_pd(x, rsqrt), rsqrt);
-    nr = _mm512_sub_pd(_mm512_set1_pd(3.0), nr);
-    return _mm512_mul_pd(_mm512_mul_pd(_mm512_set1_pd(0.5), rsqrt), nr);
+    __m512d rsqrt_f64x8 = _mm512_rsqrt14_pd(x);
+    __m512d nr_f64x8 = _mm512_mul_pd(_mm512_mul_pd(x, rsqrt_f64x8), rsqrt_f64x8);
+    nr_f64x8 = _mm512_sub_pd(_mm512_set1_pd(3.0), nr_f64x8);
+    return _mm512_mul_pd(_mm512_mul_pd(_mm512_set1_pd(0.5), rsqrt_f64x8), nr_f64x8);
 }
 
-#pragma region - Traditional Floats
+#pragma region F32 and F64 Floats
 
 NK_PUBLIC void nk_sqeuclidean_f32_skylake(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f64_t *result) {
     // Upcast to f64 for higher precision accumulation
@@ -282,8 +281,8 @@ NK_INTERNAL void nk_euclidean_through_f64_from_dot_skylake_(nk_b128_vec_t dots,
     results->xmm_ps = _mm256_cvtpd_ps(dist_f64x4);
 }
 
-#pragma endregion - Traditional Floats
-#pragma region - Smaller Floats
+#pragma endregion F32 and F64 Floats
+#pragma region F16 and BF16 Floats
 
 NK_PUBLIC void nk_sqeuclidean_f16_skylake(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *result) {
     __m512 sum_f32x16 = _mm512_setzero_ps();
@@ -348,22 +347,22 @@ nk_angular_f16_skylake_cycle:
 
 NK_PUBLIC void nk_sqeuclidean_e4m3_skylake(nk_e4m3_t const *a, nk_e4m3_t const *b, nk_size_t n, nk_f32_t *result) {
     __m512 sum_f32x16 = _mm512_setzero_ps();
-    __m128i a_e4m3x16, b_e4m3x16;
+    __m128i a_e4m3_u8x16, b_e4m3_u8x16;
 
 nk_sqeuclidean_e4m3_skylake_cycle:
     if (n < 16) {
         __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, n);
-        a_e4m3x16 = _mm_maskz_loadu_epi8(mask, a);
-        b_e4m3x16 = _mm_maskz_loadu_epi8(mask, b);
+        a_e4m3_u8x16 = _mm_maskz_loadu_epi8(mask, a);
+        b_e4m3_u8x16 = _mm_maskz_loadu_epi8(mask, b);
         n = 0;
     }
     else {
-        a_e4m3x16 = _mm_loadu_si128((__m128i const *)a);
-        b_e4m3x16 = _mm_loadu_si128((__m128i const *)b);
+        a_e4m3_u8x16 = _mm_loadu_si128((__m128i const *)a);
+        b_e4m3_u8x16 = _mm_loadu_si128((__m128i const *)b);
         a += 16, b += 16, n -= 16;
     }
-    __m512 a_f32x16 = nk_e4m3x16_to_f32x16_skylake_(a_e4m3x16);
-    __m512 b_f32x16 = nk_e4m3x16_to_f32x16_skylake_(b_e4m3x16);
+    __m512 a_f32x16 = nk_e4m3x16_to_f32x16_skylake_(a_e4m3_u8x16);
+    __m512 b_f32x16 = nk_e4m3x16_to_f32x16_skylake_(b_e4m3_u8x16);
     __m512 diff_f32x16 = _mm512_sub_ps(a_f32x16, b_f32x16);
     sum_f32x16 = _mm512_fmadd_ps(diff_f32x16, diff_f32x16, sum_f32x16);
     if (n) goto nk_sqeuclidean_e4m3_skylake_cycle;
@@ -380,22 +379,22 @@ NK_PUBLIC void nk_angular_e4m3_skylake(nk_e4m3_t const *a, nk_e4m3_t const *b, n
     __m512 dot_f32x16 = _mm512_setzero_ps();
     __m512 a_norm_sq_f32x16 = _mm512_setzero_ps();
     __m512 b_norm_sq_f32x16 = _mm512_setzero_ps();
-    __m128i a_e4m3x16, b_e4m3x16;
+    __m128i a_e4m3_u8x16, b_e4m3_u8x16;
 
 nk_angular_e4m3_skylake_cycle:
     if (n < 16) {
         __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, n);
-        a_e4m3x16 = _mm_maskz_loadu_epi8(mask, a);
-        b_e4m3x16 = _mm_maskz_loadu_epi8(mask, b);
+        a_e4m3_u8x16 = _mm_maskz_loadu_epi8(mask, a);
+        b_e4m3_u8x16 = _mm_maskz_loadu_epi8(mask, b);
         n = 0;
     }
     else {
-        a_e4m3x16 = _mm_loadu_si128((__m128i const *)a);
-        b_e4m3x16 = _mm_loadu_si128((__m128i const *)b);
+        a_e4m3_u8x16 = _mm_loadu_si128((__m128i const *)a);
+        b_e4m3_u8x16 = _mm_loadu_si128((__m128i const *)b);
         a += 16, b += 16, n -= 16;
     }
-    __m512 a_f32x16 = nk_e4m3x16_to_f32x16_skylake_(a_e4m3x16);
-    __m512 b_f32x16 = nk_e4m3x16_to_f32x16_skylake_(b_e4m3x16);
+    __m512 a_f32x16 = nk_e4m3x16_to_f32x16_skylake_(a_e4m3_u8x16);
+    __m512 b_f32x16 = nk_e4m3x16_to_f32x16_skylake_(b_e4m3_u8x16);
     dot_f32x16 = _mm512_fmadd_ps(a_f32x16, b_f32x16, dot_f32x16);
     a_norm_sq_f32x16 = _mm512_fmadd_ps(a_f32x16, a_f32x16, a_norm_sq_f32x16);
     b_norm_sq_f32x16 = _mm512_fmadd_ps(b_f32x16, b_f32x16, b_norm_sq_f32x16);
@@ -409,22 +408,22 @@ nk_angular_e4m3_skylake_cycle:
 
 NK_PUBLIC void nk_sqeuclidean_e5m2_skylake(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_f32_t *result) {
     __m512 sum_f32x16 = _mm512_setzero_ps();
-    __m128i a_e5m2x16, b_e5m2x16;
+    __m128i a_e5m2_u8x16, b_e5m2_u8x16;
 
 nk_sqeuclidean_e5m2_skylake_cycle:
     if (n < 16) {
         __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, n);
-        a_e5m2x16 = _mm_maskz_loadu_epi8(mask, a);
-        b_e5m2x16 = _mm_maskz_loadu_epi8(mask, b);
+        a_e5m2_u8x16 = _mm_maskz_loadu_epi8(mask, a);
+        b_e5m2_u8x16 = _mm_maskz_loadu_epi8(mask, b);
         n = 0;
     }
     else {
-        a_e5m2x16 = _mm_loadu_si128((__m128i const *)a);
-        b_e5m2x16 = _mm_loadu_si128((__m128i const *)b);
+        a_e5m2_u8x16 = _mm_loadu_si128((__m128i const *)a);
+        b_e5m2_u8x16 = _mm_loadu_si128((__m128i const *)b);
         a += 16, b += 16, n -= 16;
     }
-    __m512 a_f32x16 = nk_e5m2x16_to_f32x16_skylake_(a_e5m2x16);
-    __m512 b_f32x16 = nk_e5m2x16_to_f32x16_skylake_(b_e5m2x16);
+    __m512 a_f32x16 = nk_e5m2x16_to_f32x16_skylake_(a_e5m2_u8x16);
+    __m512 b_f32x16 = nk_e5m2x16_to_f32x16_skylake_(b_e5m2_u8x16);
     __m512 diff_f32x16 = _mm512_sub_ps(a_f32x16, b_f32x16);
     sum_f32x16 = _mm512_fmadd_ps(diff_f32x16, diff_f32x16, sum_f32x16);
     if (n) goto nk_sqeuclidean_e5m2_skylake_cycle;
@@ -441,22 +440,22 @@ NK_PUBLIC void nk_angular_e5m2_skylake(nk_e5m2_t const *a, nk_e5m2_t const *b, n
     __m512 dot_f32x16 = _mm512_setzero_ps();
     __m512 a_norm_sq_f32x16 = _mm512_setzero_ps();
     __m512 b_norm_sq_f32x16 = _mm512_setzero_ps();
-    __m128i a_e5m2x16, b_e5m2x16;
+    __m128i a_e5m2_u8x16, b_e5m2_u8x16;
 
 nk_angular_e5m2_skylake_cycle:
     if (n < 16) {
         __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, n);
-        a_e5m2x16 = _mm_maskz_loadu_epi8(mask, a);
-        b_e5m2x16 = _mm_maskz_loadu_epi8(mask, b);
+        a_e5m2_u8x16 = _mm_maskz_loadu_epi8(mask, a);
+        b_e5m2_u8x16 = _mm_maskz_loadu_epi8(mask, b);
         n = 0;
     }
     else {
-        a_e5m2x16 = _mm_loadu_si128((__m128i const *)a);
-        b_e5m2x16 = _mm_loadu_si128((__m128i const *)b);
+        a_e5m2_u8x16 = _mm_loadu_si128((__m128i const *)a);
+        b_e5m2_u8x16 = _mm_loadu_si128((__m128i const *)b);
         a += 16, b += 16, n -= 16;
     }
-    __m512 a_f32x16 = nk_e5m2x16_to_f32x16_skylake_(a_e5m2x16);
-    __m512 b_f32x16 = nk_e5m2x16_to_f32x16_skylake_(b_e5m2x16);
+    __m512 a_f32x16 = nk_e5m2x16_to_f32x16_skylake_(a_e5m2_u8x16);
+    __m512 b_f32x16 = nk_e5m2x16_to_f32x16_skylake_(b_e5m2_u8x16);
     dot_f32x16 = _mm512_fmadd_ps(a_f32x16, b_f32x16, dot_f32x16);
     a_norm_sq_f32x16 = _mm512_fmadd_ps(a_f32x16, a_f32x16, a_norm_sq_f32x16);
     b_norm_sq_f32x16 = _mm512_fmadd_ps(b_f32x16, b_f32x16, b_norm_sq_f32x16);
@@ -470,22 +469,22 @@ nk_angular_e5m2_skylake_cycle:
 
 NK_PUBLIC void nk_sqeuclidean_e2m3_skylake(nk_e2m3_t const *a, nk_e2m3_t const *b, nk_size_t n, nk_f32_t *result) {
     __m512 sum_f32x16 = _mm512_setzero_ps();
-    __m128i a_e2m3x16, b_e2m3x16;
+    __m128i a_e2m3_u8x16, b_e2m3_u8x16;
 
 nk_sqeuclidean_e2m3_skylake_cycle:
     if (n < 16) {
         __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, n);
-        a_e2m3x16 = _mm_maskz_loadu_epi8(mask, a);
-        b_e2m3x16 = _mm_maskz_loadu_epi8(mask, b);
+        a_e2m3_u8x16 = _mm_maskz_loadu_epi8(mask, a);
+        b_e2m3_u8x16 = _mm_maskz_loadu_epi8(mask, b);
         n = 0;
     }
     else {
-        a_e2m3x16 = _mm_loadu_si128((__m128i const *)a);
-        b_e2m3x16 = _mm_loadu_si128((__m128i const *)b);
+        a_e2m3_u8x16 = _mm_loadu_si128((__m128i const *)a);
+        b_e2m3_u8x16 = _mm_loadu_si128((__m128i const *)b);
         a += 16, b += 16, n -= 16;
     }
-    __m512 a_f32x16 = nk_e2m3x16_to_f32x16_skylake_(a_e2m3x16);
-    __m512 b_f32x16 = nk_e2m3x16_to_f32x16_skylake_(b_e2m3x16);
+    __m512 a_f32x16 = nk_e2m3x16_to_f32x16_skylake_(a_e2m3_u8x16);
+    __m512 b_f32x16 = nk_e2m3x16_to_f32x16_skylake_(b_e2m3_u8x16);
     __m512 diff_f32x16 = _mm512_sub_ps(a_f32x16, b_f32x16);
     sum_f32x16 = _mm512_fmadd_ps(diff_f32x16, diff_f32x16, sum_f32x16);
     if (n) goto nk_sqeuclidean_e2m3_skylake_cycle;
@@ -502,22 +501,22 @@ NK_PUBLIC void nk_angular_e2m3_skylake(nk_e2m3_t const *a, nk_e2m3_t const *b, n
     __m512 dot_f32x16 = _mm512_setzero_ps();
     __m512 a_norm_sq_f32x16 = _mm512_setzero_ps();
     __m512 b_norm_sq_f32x16 = _mm512_setzero_ps();
-    __m128i a_e2m3x16, b_e2m3x16;
+    __m128i a_e2m3_u8x16, b_e2m3_u8x16;
 
 nk_angular_e2m3_skylake_cycle:
     if (n < 16) {
         __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, n);
-        a_e2m3x16 = _mm_maskz_loadu_epi8(mask, a);
-        b_e2m3x16 = _mm_maskz_loadu_epi8(mask, b);
+        a_e2m3_u8x16 = _mm_maskz_loadu_epi8(mask, a);
+        b_e2m3_u8x16 = _mm_maskz_loadu_epi8(mask, b);
         n = 0;
     }
     else {
-        a_e2m3x16 = _mm_loadu_si128((__m128i const *)a);
-        b_e2m3x16 = _mm_loadu_si128((__m128i const *)b);
+        a_e2m3_u8x16 = _mm_loadu_si128((__m128i const *)a);
+        b_e2m3_u8x16 = _mm_loadu_si128((__m128i const *)b);
         a += 16, b += 16, n -= 16;
     }
-    __m512 a_f32x16 = nk_e2m3x16_to_f32x16_skylake_(a_e2m3x16);
-    __m512 b_f32x16 = nk_e2m3x16_to_f32x16_skylake_(b_e2m3x16);
+    __m512 a_f32x16 = nk_e2m3x16_to_f32x16_skylake_(a_e2m3_u8x16);
+    __m512 b_f32x16 = nk_e2m3x16_to_f32x16_skylake_(b_e2m3_u8x16);
     dot_f32x16 = _mm512_fmadd_ps(a_f32x16, b_f32x16, dot_f32x16);
     a_norm_sq_f32x16 = _mm512_fmadd_ps(a_f32x16, a_f32x16, a_norm_sq_f32x16);
     b_norm_sq_f32x16 = _mm512_fmadd_ps(b_f32x16, b_f32x16, b_norm_sq_f32x16);
@@ -531,22 +530,22 @@ nk_angular_e2m3_skylake_cycle:
 
 NK_PUBLIC void nk_sqeuclidean_e3m2_skylake(nk_e3m2_t const *a, nk_e3m2_t const *b, nk_size_t n, nk_f32_t *result) {
     __m512 sum_f32x16 = _mm512_setzero_ps();
-    __m128i a_e3m2x16, b_e3m2x16;
+    __m128i a_e3m2_u8x16, b_e3m2_u8x16;
 
 nk_sqeuclidean_e3m2_skylake_cycle:
     if (n < 16) {
         __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, n);
-        a_e3m2x16 = _mm_maskz_loadu_epi8(mask, a);
-        b_e3m2x16 = _mm_maskz_loadu_epi8(mask, b);
+        a_e3m2_u8x16 = _mm_maskz_loadu_epi8(mask, a);
+        b_e3m2_u8x16 = _mm_maskz_loadu_epi8(mask, b);
         n = 0;
     }
     else {
-        a_e3m2x16 = _mm_loadu_si128((__m128i const *)a);
-        b_e3m2x16 = _mm_loadu_si128((__m128i const *)b);
+        a_e3m2_u8x16 = _mm_loadu_si128((__m128i const *)a);
+        b_e3m2_u8x16 = _mm_loadu_si128((__m128i const *)b);
         a += 16, b += 16, n -= 16;
     }
-    __m512 a_f32x16 = nk_e3m2x16_to_f32x16_skylake_(a_e3m2x16);
-    __m512 b_f32x16 = nk_e3m2x16_to_f32x16_skylake_(b_e3m2x16);
+    __m512 a_f32x16 = nk_e3m2x16_to_f32x16_skylake_(a_e3m2_u8x16);
+    __m512 b_f32x16 = nk_e3m2x16_to_f32x16_skylake_(b_e3m2_u8x16);
     __m512 diff_f32x16 = _mm512_sub_ps(a_f32x16, b_f32x16);
     sum_f32x16 = _mm512_fmadd_ps(diff_f32x16, diff_f32x16, sum_f32x16);
     if (n) goto nk_sqeuclidean_e3m2_skylake_cycle;
@@ -563,22 +562,22 @@ NK_PUBLIC void nk_angular_e3m2_skylake(nk_e3m2_t const *a, nk_e3m2_t const *b, n
     __m512 dot_f32x16 = _mm512_setzero_ps();
     __m512 a_norm_sq_f32x16 = _mm512_setzero_ps();
     __m512 b_norm_sq_f32x16 = _mm512_setzero_ps();
-    __m128i a_e3m2x16, b_e3m2x16;
+    __m128i a_e3m2_u8x16, b_e3m2_u8x16;
 
 nk_angular_e3m2_skylake_cycle:
     if (n < 16) {
         __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, n);
-        a_e3m2x16 = _mm_maskz_loadu_epi8(mask, a);
-        b_e3m2x16 = _mm_maskz_loadu_epi8(mask, b);
+        a_e3m2_u8x16 = _mm_maskz_loadu_epi8(mask, a);
+        b_e3m2_u8x16 = _mm_maskz_loadu_epi8(mask, b);
         n = 0;
     }
     else {
-        a_e3m2x16 = _mm_loadu_si128((__m128i const *)a);
-        b_e3m2x16 = _mm_loadu_si128((__m128i const *)b);
+        a_e3m2_u8x16 = _mm_loadu_si128((__m128i const *)a);
+        b_e3m2_u8x16 = _mm_loadu_si128((__m128i const *)b);
         a += 16, b += 16, n -= 16;
     }
-    __m512 a_f32x16 = nk_e3m2x16_to_f32x16_skylake_(a_e3m2x16);
-    __m512 b_f32x16 = nk_e3m2x16_to_f32x16_skylake_(b_e3m2x16);
+    __m512 a_f32x16 = nk_e3m2x16_to_f32x16_skylake_(a_e3m2_u8x16);
+    __m512 b_f32x16 = nk_e3m2x16_to_f32x16_skylake_(b_e3m2_u8x16);
     dot_f32x16 = _mm512_fmadd_ps(a_f32x16, b_f32x16, dot_f32x16);
     a_norm_sq_f32x16 = _mm512_fmadd_ps(a_f32x16, a_f32x16, a_norm_sq_f32x16);
     b_norm_sq_f32x16 = _mm512_fmadd_ps(b_f32x16, b_f32x16, b_norm_sq_f32x16);
@@ -600,7 +599,7 @@ nk_angular_e3m2_skylake_cycle:
 } // extern "C"
 #endif
 
-#pragma endregion - Smaller Floats
+#pragma endregion F16 and BF16 Floats
 #endif // NK_TARGET_SKYLAKE
 #endif // NK_TARGET_X86_
 #endif // NK_SPATIAL_SKYLAKE_H

package/include/numkong/spatial/sve.h

@@ -8,19 +8,19 @@
  *
  *  @section spatial_sve_instructions ARM SVE Instructions
  *
- *      Intrinsic                   Instruction                     Latency     Throughput
- *      svld1_f32                   LD1W (Z.S, P/Z, [Xn])           4-6cy       2/cy
- *      svsub_f32_x                 FSUB (Z.S, P/M, Z.S, Z.S)       3cy         2/cy
- *      svmla_f32_x                 FMLA (Z.S, P/M, Z.S, Z.S)       4cy         2/cy
- *      svaddv_f32                  FADDV (S, P, Z.S)               6cy         1/cy
- *      svdupq_n_f32                DUP (Z.S, #imm)                 1cy         2/cy
- *      svwhilelt_b32               WHILELT (P.S, Xn, Xm)           2cy         1/cy
- *      svptrue_b32                 PTRUE (P.S, pattern)            1cy         2/cy
- *      svcntw                      CNTW (Xd)                       1cy         2/cy
- *      svld1_f64                   LD1D (Z.D, P/Z, [Xn])           4-6cy       2/cy
- *      svsub_f64_x                 FSUB (Z.D, P/M, Z.D, Z.D)       3cy         2/cy
- *      svmla_f64_x                 FMLA (Z.D, P/M, Z.D, Z.D)       4cy         2/cy
- *      svaddv_f64                  FADDV (D, P, Z.D)               6cy         1/cy
+ *      Intrinsic      Instruction                V1
+ *      svld1_f32      LD1W (Z.S, P/Z, [Xn])      4-6cy @ 2p
+ *      svsub_f32_x    FSUB (Z.S, P/M, Z.S, Z.S)  3cy @ 2p
+ *      svmla_f32_x    FMLA (Z.S, P/M, Z.S, Z.S)  4cy @ 2p
+ *      svaddv_f32     FADDV (S, P, Z.S)          6cy @ 1p
+ *      svdupq_n_f32   DUP (Z.S, #imm)            1cy @ 2p
+ *      svwhilelt_b32  WHILELT (P.S, Xn, Xm)      2cy @ 1p
+ *      svptrue_b32    PTRUE (P.S, pattern)       1cy @ 2p
+ *      svcntw         CNTW (Xd)                  1cy @ 2p
+ *      svld1_f64      LD1D (Z.D, P/Z, [Xn])      4-6cy @ 2p
+ *      svsub_f64_x    FSUB (Z.D, P/M, Z.D, Z.D)  3cy @ 2p
+ *      svmla_f64_x    FMLA (Z.D, P/M, Z.D, Z.D)  4cy @ 2p
+ *      svaddv_f64     FADDV (D, P, Z.D)          6cy @ 1p
  *
  *  SVE vector widths vary across implementations: Graviton3 uses 256-bit, while Graviton4/5
  *  and Apple M4+ use 128-bit. Code using svcntb() adapts automatically, but wider vectors
@@ -63,10 +63,10 @@ extern "C" {
  *  @param x         Input vector (must be positive for meaningful results)
  *  @return          Approximate 1/sqrt(x) with ~23-bit mantissa accuracy
  */
-NK_INTERNAL svfloat32_t nk_rsqrt_f32x_sve_(svbool_t predicate, svfloat32_t x) NK_STREAMING_COMPATIBLE_ {
+NK_INTERNAL svfloat32_t nk_rsqrt_f32x_sve_(svbool_t predicate_b32x, svfloat32_t x) NK_STREAMING_COMPATIBLE_ {
     svfloat32_t r = svrsqrte_f32(x);
-    r = svmul_f32_x(predicate, r, svrsqrts_f32(svmul_f32_x(predicate, x, r), r));
-    r = svmul_f32_x(predicate, r, svrsqrts_f32(svmul_f32_x(predicate, x, r), r));
+    r = svmul_f32_x(predicate_b32x, r, svrsqrts_f32(svmul_f32_x(predicate_b32x, x, r), r));
+    r = svmul_f32_x(predicate_b32x, r, svrsqrts_f32(svmul_f32_x(predicate_b32x, x, r), r));
     return r;
 }
 
@@ -79,29 +79,39 @@ NK_INTERNAL svfloat32_t nk_rsqrt_f32x_sve_(svbool_t predicate, svfloat32_t x) NK
  *  Marked `__arm_streaming_compatible` so the helper is callable from both streaming
  *  (SME) and non-streaming (SVE) contexts without mode transitions.
  *
- *  @param predicate Active-lane mask
+ *  @param predicate_b32x Active-lane mask
  *  @param x         Input vector (must be positive for meaningful results)
  *  @return          Approximate 1/sqrt(x) with ~52-bit mantissa accuracy
  */
-NK_INTERNAL svfloat64_t nk_rsqrt_f64x_sve_(svbool_t predicate, svfloat64_t x) NK_STREAMING_COMPATIBLE_ {
+NK_INTERNAL svfloat64_t nk_rsqrt_f64x_sve_(svbool_t predicate_b64x, svfloat64_t x) NK_STREAMING_COMPATIBLE_ {
     svfloat64_t r = svrsqrte_f64(x);
-    r = svmul_f64_x(predicate, r, svrsqrts_f64(svmul_f64_x(predicate, x, r), r));
-    r = svmul_f64_x(predicate, r, svrsqrts_f64(svmul_f64_x(predicate, x, r), r));
-    r = svmul_f64_x(predicate, r, svrsqrts_f64(svmul_f64_x(predicate, x, r), r));
+    r = svmul_f64_x(predicate_b64x, r, svrsqrts_f64(svmul_f64_x(predicate_b64x, x, r), r));
+    r = svmul_f64_x(predicate_b64x, r, svrsqrts_f64(svmul_f64_x(predicate_b64x, x, r), r));
+    r = svmul_f64_x(predicate_b64x, r, svrsqrts_f64(svmul_f64_x(predicate_b64x, x, r), r));
     return r;
 }
 
 NK_PUBLIC void nk_sqeuclidean_f32_sve(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f64_t *result) {
     nk_size_t i = 0;
-    nk_size_t const vector_length = svcntd();
     svfloat64_t dist_sq_f64x = svdupq_n_f64(0.0, 0.0);
-    for (; i < n; i += vector_length) {
-        svbool_t predicate_f32x = svwhilelt_b32_u64(i, n);
-        svbool_t predicate_f64x = svwhilelt_b64_u64(i, n);
-        svfloat64_t a_f64x = svcvt_f64_f32_x(predicate_f64x, svld1_f32(predicate_f32x, a + i));
-        svfloat64_t b_f64x = svcvt_f64_f32_x(predicate_f64x, svld1_f32(predicate_f32x, b + i));
-        svfloat64_t diff_f64x = svsub_f64_x(predicate_f64x, a_f64x, b_f64x);
-        dist_sq_f64x = svmla_f64_x(predicate_f64x, dist_sq_f64x, diff_f64x, diff_f64x);
+    for (; i < n; i += svcntw()) {
+        svbool_t predicate_b32x = svwhilelt_b32_u64(i, n);
+        svfloat32_t a_f32x = svld1_f32(predicate_b32x, a + i);
+        svfloat32_t b_f32x = svld1_f32(predicate_b32x, b + i);
+        nk_size_t remaining = n - i < svcntw() ? n - i : svcntw();
+
+        // svcvt_f64_f32_x widens only even-indexed f32 elements; svext by 1 shifts odd into even.
+        svbool_t pred_even_b64x = svwhilelt_b64_u64(0u, (remaining + 1) / 2);
+        svfloat64_t a_even_f64x = svcvt_f64_f32_x(pred_even_b64x, a_f32x);
+        svfloat64_t b_even_f64x = svcvt_f64_f32_x(pred_even_b64x, b_f32x);
+        svfloat64_t diff_even_f64x = svsub_f64_x(pred_even_b64x, a_even_f64x, b_even_f64x);
+        dist_sq_f64x = svmla_f64_m(pred_even_b64x, dist_sq_f64x, diff_even_f64x, diff_even_f64x);
+
+        svbool_t pred_odd_b64x = svwhilelt_b64_u64(0u, remaining / 2);
+        svfloat64_t a_odd_f64x = svcvt_f64_f32_x(pred_odd_b64x, svext_f32(a_f32x, a_f32x, 1));
+        svfloat64_t b_odd_f64x = svcvt_f64_f32_x(pred_odd_b64x, svext_f32(b_f32x, b_f32x, 1));
+        svfloat64_t diff_odd_f64x = svsub_f64_x(pred_odd_b64x, a_odd_f64x, b_odd_f64x);
+        dist_sq_f64x = svmla_f64_m(pred_odd_b64x, dist_sq_f64x, diff_odd_f64x, diff_odd_f64x);
     }
     nk_f64_t dist_sq_f64 = svaddv_f64(svptrue_b64(), dist_sq_f64x);
     *result = dist_sq_f64;
@@ -114,18 +124,29 @@ NK_PUBLIC void nk_euclidean_f32_sve(nk_f32_t const *a, nk_f32_t const *b, nk_siz
 
 NK_PUBLIC void nk_angular_f32_sve(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f64_t *result) {
     nk_size_t i = 0;
-    nk_size_t const vector_length = svcntd();
     svfloat64_t ab_f64x = svdupq_n_f64(0.0, 0.0);
     svfloat64_t a2_f64x = svdupq_n_f64(0.0, 0.0);
     svfloat64_t b2_f64x = svdupq_n_f64(0.0, 0.0);
-    for (; i < n; i += vector_length) {
-        svbool_t predicate_f32x = svwhilelt_b32_u64(i, n);
-        svbool_t predicate_f64x = svwhilelt_b64_u64(i, n);
-        svfloat64_t a_f64x = svcvt_f64_f32_x(predicate_f64x, svld1_f32(predicate_f32x, a + i));
-        svfloat64_t b_f64x = svcvt_f64_f32_x(predicate_f64x, svld1_f32(predicate_f32x, b + i));
-        ab_f64x = svmla_f64_x(predicate_f64x, ab_f64x, a_f64x, b_f64x);
-        a2_f64x = svmla_f64_x(predicate_f64x, a2_f64x, a_f64x, a_f64x);
-        b2_f64x = svmla_f64_x(predicate_f64x, b2_f64x, b_f64x, b_f64x);
+    for (; i < n; i += svcntw()) {
+        svbool_t predicate_b32x = svwhilelt_b32_u64(i, n);
+        svfloat32_t a_f32x = svld1_f32(predicate_b32x, a + i);
+        svfloat32_t b_f32x = svld1_f32(predicate_b32x, b + i);
+        nk_size_t remaining = n - i < svcntw() ? n - i : svcntw();
+
+        // svcvt_f64_f32_x widens only even-indexed f32 elements; svext by 1 shifts odd into even.
+        svbool_t pred_even_b64x = svwhilelt_b64_u64(0u, (remaining + 1) / 2);
+        svfloat64_t a_even_f64x = svcvt_f64_f32_x(pred_even_b64x, a_f32x);
+        svfloat64_t b_even_f64x = svcvt_f64_f32_x(pred_even_b64x, b_f32x);
+        ab_f64x = svmla_f64_m(pred_even_b64x, ab_f64x, a_even_f64x, b_even_f64x);
+        a2_f64x = svmla_f64_m(pred_even_b64x, a2_f64x, a_even_f64x, a_even_f64x);
+        b2_f64x = svmla_f64_m(pred_even_b64x, b2_f64x, b_even_f64x, b_even_f64x);
+
+        svbool_t pred_odd_b64x = svwhilelt_b64_u64(0u, remaining / 2);
+        svfloat64_t a_odd_f64x = svcvt_f64_f32_x(pred_odd_b64x, svext_f32(a_f32x, a_f32x, 1));
+        svfloat64_t b_odd_f64x = svcvt_f64_f32_x(pred_odd_b64x, svext_f32(b_f32x, b_f32x, 1));
+        ab_f64x = svmla_f64_m(pred_odd_b64x, ab_f64x, a_odd_f64x, b_odd_f64x);
+        a2_f64x = svmla_f64_m(pred_odd_b64x, a2_f64x, a_odd_f64x, a_odd_f64x);
+        b2_f64x = svmla_f64_m(pred_odd_b64x, b2_f64x, b_odd_f64x, b_odd_f64x);
     }
 
     nk_f64_t ab_f64 = svaddv_f64(svptrue_b64(), ab_f64x);
@@ -139,29 +160,29 @@ NK_PUBLIC void nk_sqeuclidean_f64_sve(nk_f64_t const *a, nk_f64_t const *b, nk_s
     nk_size_t i = 0;
     svfloat64_t sum_f64x = svdupq_n_f64(0.0, 0.0);
     svfloat64_t compensation_f64x = svdupq_n_f64(0.0, 0.0);
-    svbool_t predicate_all_f64x = svptrue_b64();
+    svbool_t predicate_all_b64x = svptrue_b64();
     do {
-        svbool_t predicate_f64x = svwhilelt_b64_u64(i, n);
-        svfloat64_t a_f64x = svld1_f64(predicate_f64x, a + i);
-        svfloat64_t b_f64x = svld1_f64(predicate_f64x, b + i);
-        svfloat64_t diff_f64x = svsub_f64_x(predicate_f64x, a_f64x, b_f64x);
-        svfloat64_t diff_sq_f64x = svmul_f64_x(predicate_f64x, diff_f64x, diff_f64x);
+        svbool_t predicate_b64x = svwhilelt_b64_u64(i, n);
+        svfloat64_t a_f64x = svld1_f64(predicate_b64x, a + i);
+        svfloat64_t b_f64x = svld1_f64(predicate_b64x, b + i);
+        svfloat64_t diff_f64x = svsub_f64_x(predicate_b64x, a_f64x, b_f64x);
+        svfloat64_t diff_sq_f64x = svmul_f64_x(predicate_b64x, diff_f64x, diff_f64x);
         // Neumaier: t = sum + x
-        svfloat64_t t_f64x = svadd_f64_x(predicate_f64x, sum_f64x, diff_sq_f64x);
-        svfloat64_t abs_sum_f64x = svabs_f64_x(predicate_f64x, sum_f64x);
+        svfloat64_t t_f64x = svadd_f64_m(predicate_b64x, sum_f64x, diff_sq_f64x);
+        svfloat64_t abs_sum_f64x = svabs_f64_x(predicate_b64x, sum_f64x);
         // diff_sq is already non-negative (it's a square), so svabs is unnecessary
-        svbool_t sum_ge_x_f64x = svcmpge_f64(predicate_f64x, abs_sum_f64x, diff_sq_f64x);
+        svbool_t sum_ge_x_b64x = svcmpge_f64(predicate_b64x, abs_sum_f64x, diff_sq_f64x);
         // When |sum| >= |x|: comp += (sum - t) + x; when |x| > |sum|: comp += (x - t) + sum
-        svfloat64_t comp_sum_large_f64x = svadd_f64_x(predicate_f64x, svsub_f64_x(predicate_f64x, sum_f64x, t_f64x),
+        svfloat64_t comp_sum_large_f64x = svadd_f64_x(predicate_b64x, svsub_f64_x(predicate_b64x, sum_f64x, t_f64x),
                                                       diff_sq_f64x);
-        svfloat64_t comp_x_large_f64x = svadd_f64_x(predicate_f64x, svsub_f64_x(predicate_f64x, diff_sq_f64x, t_f64x),
+        svfloat64_t comp_x_large_f64x = svadd_f64_x(predicate_b64x, svsub_f64_x(predicate_b64x, diff_sq_f64x, t_f64x),
                                                     sum_f64x);
-        svfloat64_t comp_update_f64x = svsel_f64(sum_ge_x_f64x, comp_sum_large_f64x, comp_x_large_f64x);
-        compensation_f64x = svadd_f64_x(predicate_f64x, compensation_f64x, comp_update_f64x);
+        svfloat64_t comp_update_f64x = svsel_f64(sum_ge_x_b64x, comp_sum_large_f64x, comp_x_large_f64x);
+        compensation_f64x = svadd_f64_m(predicate_b64x, compensation_f64x, comp_update_f64x);
         sum_f64x = t_f64x;
         i += svcntd();
     } while (i < n);
-    *result = nk_dot_stable_sum_f64_sve_(predicate_all_f64x, sum_f64x, compensation_f64x);
+    *result = nk_dot_stable_sum_f64_sve_(predicate_all_b64x, sum_f64x, compensation_f64x);
 }
 
 NK_PUBLIC void nk_euclidean_f64_sve(nk_f64_t const *a, nk_f64_t const *b, nk_size_t n, nk_f64_t *result) {
@@ -177,35 +198,35 @@ NK_PUBLIC void nk_angular_f64_sve(nk_f64_t const *a, nk_f64_t const *b, nk_size_
     svfloat64_t ab_compensation_f64x = svdupq_n_f64(0.0, 0.0);
     svfloat64_t a2_f64x = svdupq_n_f64(0.0, 0.0);
     svfloat64_t b2_f64x = svdupq_n_f64(0.0, 0.0);
-    svbool_t predicate_all_f64x = svptrue_b64();
+    svbool_t predicate_all_b64x = svptrue_b64();
     do {
-        svbool_t predicate_f64x = svwhilelt_b64_u64(i, n);
-        svfloat64_t a_f64x = svld1_f64(predicate_f64x, a + i);
-        svfloat64_t b_f64x = svld1_f64(predicate_f64x, b + i);
+        svbool_t predicate_b64x = svwhilelt_b64_u64(i, n);
+        svfloat64_t a_f64x = svld1_f64(predicate_b64x, a + i);
+        svfloat64_t b_f64x = svld1_f64(predicate_b64x, b + i);
         // TwoProd for ab: product = a*b, error = fma(a,b,-product) = -(product - a*b)
-        svfloat64_t product_f64x = svmul_f64_x(predicate_f64x, a_f64x, b_f64x);
-        svfloat64_t product_error_f64x = svneg_f64_x(predicate_f64x,
-                                                     svnmls_f64_x(predicate_f64x, product_f64x, a_f64x, b_f64x));
+        svfloat64_t product_f64x = svmul_f64_x(predicate_b64x, a_f64x, b_f64x);
+        svfloat64_t product_error_f64x = svneg_f64_x(predicate_b64x,
+                                                     svnmls_f64_x(predicate_b64x, product_f64x, a_f64x, b_f64x));
         // TwoSum: (tentative_sum, sum_error) = TwoSum(sum, product)
-        svfloat64_t tentative_sum_f64x = svadd_f64_x(predicate_f64x, ab_sum_f64x, product_f64x);
-        svfloat64_t virtual_addend_f64x = svsub_f64_x(predicate_f64x, tentative_sum_f64x, ab_sum_f64x);
+        svfloat64_t tentative_sum_f64x = svadd_f64_m(predicate_b64x, ab_sum_f64x, product_f64x);
+        svfloat64_t virtual_addend_f64x = svsub_f64_x(predicate_b64x, tentative_sum_f64x, ab_sum_f64x);
         svfloat64_t sum_error_f64x = svadd_f64_x(
-            predicate_f64x,
-            svsub_f64_x(predicate_f64x, ab_sum_f64x,
-                        svsub_f64_x(predicate_f64x, tentative_sum_f64x, virtual_addend_f64x)),
-            svsub_f64_x(predicate_f64x, product_f64x, virtual_addend_f64x));
+            predicate_b64x,
+            svsub_f64_x(predicate_b64x, ab_sum_f64x,
+                        svsub_f64_x(predicate_b64x, tentative_sum_f64x, virtual_addend_f64x)),
+            svsub_f64_x(predicate_b64x, product_f64x, virtual_addend_f64x));
         ab_sum_f64x = tentative_sum_f64x;
-        ab_compensation_f64x = svadd_f64_x(predicate_f64x, ab_compensation_f64x,
-                                           svadd_f64_x(predicate_f64x, sum_error_f64x, product_error_f64x));
+        ab_compensation_f64x = svadd_f64_m(predicate_b64x, ab_compensation_f64x,
+                                           svadd_f64_x(predicate_b64x, sum_error_f64x, product_error_f64x));
         // Simple FMA for self-products (no cancellation)
-        a2_f64x = svmla_f64_x(predicate_f64x, a2_f64x, a_f64x, a_f64x);
-        b2_f64x = svmla_f64_x(predicate_f64x, b2_f64x, b_f64x, b_f64x);
+        a2_f64x = svmla_f64_m(predicate_b64x, a2_f64x, a_f64x, a_f64x);
+        b2_f64x = svmla_f64_m(predicate_b64x, b2_f64x, b_f64x, b_f64x);
         i += svcntd();
     } while (i < n);
 
-    nk_f64_t ab_f64 = nk_dot_stable_sum_f64_sve_(predicate_all_f64x, ab_sum_f64x, ab_compensation_f64x);
-    nk_f64_t a2_f64 = svaddv_f64(predicate_all_f64x, a2_f64x);
-    nk_f64_t b2_f64 = svaddv_f64(predicate_all_f64x, b2_f64x);
+    nk_f64_t ab_f64 = nk_dot_stable_sum_f64_sve_(predicate_all_b64x, ab_sum_f64x, ab_compensation_f64x);
+    nk_f64_t a2_f64 = svaddv_f64(predicate_all_b64x, a2_f64x);
+    nk_f64_t b2_f64 = svaddv_f64(predicate_all_b64x, b2_f64x);
     *result = nk_angular_normalize_f64_neon_(ab_f64, a2_f64, b2_f64);
 }
 

package/include/numkong/spatial/svebfdot.h

@@ -8,19 +8,19 @@
  *
  *  @section spatial_svebfdot_instructions ARM SVE+BF16 Instructions
  *
- *      Intrinsic                   Instruction                     Latency     Throughput
- *      svld1_bf16                  LD1H (Z.H, P/Z, [Xn])           4-6cy       2/cy
- *      svld1_u16                   LD1H (Z.H, P/Z, [Xn])           4-6cy       2/cy
- *      svbfdot_f32                 BFDOT (Z.S, Z.H, Z.H)           4cy         2/cy
- *      svmla_f32_x                 FMLA (Z.S, P/M, Z.S, Z.S)       4cy         2/cy
- *      svsub_f32_x                 FSUB (Z.S, P/M, Z.S, Z.S)       3cy         2/cy
- *      svaddv_f32                  FADDV (S, P, Z.S)               6cy         1/cy
- *      svunpklo_u32                UUNPKLO (Z.S, Z.H)              2cy         2/cy
- *      svunpkhi_u32                UUNPKHI (Z.S, Z.H)              2cy         2/cy
- *      svlsl_n_u32_x               LSL (Z.S, P/M, Z.S, #imm)       2cy         2/cy
- *      svwhilelt_b16               WHILELT (P.H, Xn, Xm)           2cy         1/cy
- *      svwhilelt_b32               WHILELT (P.S, Xn, Xm)           2cy         1/cy
- *      svcnth                      CNTH (Xd)                       1cy         2/cy
+ *      Intrinsic      Instruction                V1
+ *      svld1_bf16     LD1H (Z.H, P/Z, [Xn])      4-6cy @ 2p
+ *      svld1_u16      LD1H (Z.H, P/Z, [Xn])      4-6cy @ 2p
+ *      svbfdot_f32    BFDOT (Z.S, Z.H, Z.H)      4cy @ 2p
+ *      svmla_f32_x    FMLA (Z.S, P/M, Z.S, Z.S)  4cy @ 2p
+ *      svsub_f32_x    FSUB (Z.S, P/M, Z.S, Z.S)  3cy @ 2p
+ *      svaddv_f32     FADDV (S, P, Z.S)          6cy @ 1p
+ *      svunpklo_u32   UUNPKLO (Z.S, Z.H)         2cy @ 2p
+ *      svunpkhi_u32   UUNPKHI (Z.S, Z.H)         2cy @ 2p
+ *      svlsl_n_u32_x  LSL (Z.S, P/M, Z.S, #imm)  2cy @ 2p
+ *      svwhilelt_b16  WHILELT (P.H, Xn, Xm)      2cy @ 1p
+ *      svwhilelt_b32  WHILELT (P.S, Xn, Xm)      2cy @ 1p
+ *      svcnth         CNTH (Xd)                  1cy @ 2p
  *
  *  SVE vector widths vary across implementations: Graviton3 uses 256-bit, while Graviton4/5
  *  and Apple M4+ use 128-bit. Code using svcntb() adapts automatically, but wider vectors
@@ -57,22 +57,22 @@ NK_PUBLIC void nk_sqeuclidean_bf16_svebfdot(nk_bf16_t const *a_enum, nk_bf16_t c
     nk_u16_t const *a = (nk_u16_t const *)(a_enum);
     nk_u16_t const *b = (nk_u16_t const *)(b_enum);
     do {
-        svbool_t predicate_bf16x = svwhilelt_b16_u64(i, n);
-        svuint16_t a_u16x = svld1_u16(predicate_bf16x, a + i);
-        svuint16_t b_u16x = svld1_u16(predicate_bf16x, b + i);
+        svbool_t predicate_b16x = svwhilelt_b16_u64(i, n);
+        svuint16_t a_u16x = svld1_u16(predicate_b16x, a + i);
+        svuint16_t b_u16x = svld1_u16(predicate_b16x, b + i);
 
         // There is no `bf16` subtraction in SVE, so we need to convert to `u32` and shift.
-        svbool_t predicate_low_f32x = svwhilelt_b32_u64(i, n);
-        svbool_t predicate_high_f32x = svwhilelt_b32_u64(i + svcnth() / 2, n);
-        svfloat32_t a_low_f32x = svreinterpret_f32_u32(svlsl_n_u32_x(predicate_low_f32x, svunpklo_u32(a_u16x), 16));
-        svfloat32_t a_high_f32x = svreinterpret_f32_u32(svlsl_n_u32_x(predicate_high_f32x, svunpkhi_u32(a_u16x), 16));
-        svfloat32_t b_low_f32x = svreinterpret_f32_u32(svlsl_n_u32_x(predicate_low_f32x, svunpklo_u32(b_u16x), 16));
-        svfloat32_t b_high_f32x = svreinterpret_f32_u32(svlsl_n_u32_x(predicate_high_f32x, svunpkhi_u32(b_u16x), 16));
+        svbool_t predicate_low_b32x = svwhilelt_b32_u64(i, n);
+        svbool_t predicate_high_b32x = svwhilelt_b32_u64(i + svcnth() / 2, n);
+        svfloat32_t a_low_f32x = svreinterpret_f32_u32(svlsl_n_u32_x(predicate_low_b32x, svunpklo_u32(a_u16x), 16));
+        svfloat32_t a_high_f32x = svreinterpret_f32_u32(svlsl_n_u32_x(predicate_high_b32x, svunpkhi_u32(a_u16x), 16));
+        svfloat32_t b_low_f32x = svreinterpret_f32_u32(svlsl_n_u32_x(predicate_low_b32x, svunpklo_u32(b_u16x), 16));
+        svfloat32_t b_high_f32x = svreinterpret_f32_u32(svlsl_n_u32_x(predicate_high_b32x, svunpkhi_u32(b_u16x), 16));
 
-        svfloat32_t a_minus_b_low_f32x = svsub_f32_x(predicate_low_f32x, a_low_f32x, b_low_f32x);
-        svfloat32_t a_minus_b_high_f32x = svsub_f32_x(predicate_high_f32x, a_high_f32x, b_high_f32x);
-        d2_low_f32x = svmla_f32_x(predicate_bf16x, d2_low_f32x, a_minus_b_low_f32x, a_minus_b_low_f32x);
-        d2_high_f32x = svmla_f32_x(predicate_bf16x, d2_high_f32x, a_minus_b_high_f32x, a_minus_b_high_f32x);
+        svfloat32_t a_minus_b_low_f32x = svsub_f32_x(predicate_low_b32x, a_low_f32x, b_low_f32x);
+        svfloat32_t a_minus_b_high_f32x = svsub_f32_x(predicate_high_b32x, a_high_f32x, b_high_f32x);
+        d2_low_f32x = svmla_f32_m(predicate_low_b32x, d2_low_f32x, a_minus_b_low_f32x, a_minus_b_low_f32x);
+        d2_high_f32x = svmla_f32_m(predicate_high_b32x, d2_high_f32x, a_minus_b_high_f32x, a_minus_b_high_f32x);
         i += svcnth();
     } while (i < n);
     nk_f32_t d2 = svaddv_f32(svptrue_b32(), d2_low_f32x) + svaddv_f32(svptrue_b32(), d2_high_f32x);
@@ -92,9 +92,9 @@ NK_PUBLIC void nk_angular_bf16_svebfdot(nk_bf16_t const *a_enum, nk_bf16_t const
     nk_bf16_for_arm_simd_t const *a = (nk_bf16_for_arm_simd_t const *)(a_enum);
     nk_bf16_for_arm_simd_t const *b = (nk_bf16_for_arm_simd_t const *)(b_enum);
     do {
-        svbool_t predicate_bf16x = svwhilelt_b16_u64(i, n);
-        svbfloat16_t a_bf16x = svld1_bf16(predicate_bf16x, a + i);
-        svbfloat16_t b_bf16x = svld1_bf16(predicate_bf16x, b + i);
+        svbool_t predicate_b16x = svwhilelt_b16_u64(i, n);
+        svbfloat16_t a_bf16x = svld1_bf16(predicate_b16x, a + i);
+        svbfloat16_t b_bf16x = svld1_bf16(predicate_b16x, b + i);
         ab_f32x = svbfdot_f32(ab_f32x, a_bf16x, b_bf16x);
         a2_f32x = svbfdot_f32(a2_f32x, a_bf16x, a_bf16x);
         b2_f32x = svbfdot_f32(b2_f32x, b_bf16x, b_bf16x);