numkong 7.0.0 → 7.4.1

package/include/numkong/spatial/rvvhalf.h

@@ -34,8 +34,8 @@ extern "C" {
 NK_PUBLIC void nk_sqeuclidean_f16_rvvhalf(nk_f16_t const *a_scalars, nk_f16_t const *b_scalars, nk_size_t count_scalars,
                                           nk_f32_t *result) {
     // Per-lane accumulator — deferred horizontal reduction
-    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
-    vfloat32m2_t sum_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
+    nk_size_t max_vector_length = __riscv_vsetvlmax_e32m2();
+    vfloat32m2_t sum_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, max_vector_length);
 
     for (nk_size_t vector_length; count_scalars > 0;
          count_scalars -= vector_length, a_scalars += vector_length, b_scalars += vector_length) {
@@ -54,7 +54,7 @@ NK_PUBLIC void nk_sqeuclidean_f16_rvvhalf(nk_f16_t const *a_scalars, nk_f16_t co
 
     // Single horizontal reduction after the loop
     vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
-    *result = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m2_f32m1(sum_f32m2, zero_f32m1, vlmax));
+    *result = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m2_f32m1(sum_f32m2, zero_f32m1, max_vector_length));
 }
 
 NK_PUBLIC void nk_euclidean_f16_rvvhalf(nk_f16_t const *a_scalars, nk_f16_t const *b_scalars, nk_size_t count_scalars,
@@ -66,10 +66,10 @@ NK_PUBLIC void nk_euclidean_f16_rvvhalf(nk_f16_t const *a_scalars, nk_f16_t cons
 NK_PUBLIC void nk_angular_f16_rvvhalf(nk_f16_t const *a_scalars, nk_f16_t const *b_scalars, nk_size_t count_scalars,
                                       nk_f32_t *result) {
     // Per-lane accumulators — deferred horizontal reduction
-    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
-    vfloat32m2_t dot_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
-    vfloat32m2_t a_sq_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
-    vfloat32m2_t b_sq_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
+    nk_size_t max_vector_length = __riscv_vsetvlmax_e32m2();
+    vfloat32m2_t dot_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, max_vector_length);
+    vfloat32m2_t a_sq_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, max_vector_length);
+    vfloat32m2_t b_sq_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, max_vector_length);
 
     for (nk_size_t vector_length; count_scalars > 0;
          count_scalars -= vector_length, a_scalars += vector_length, b_scalars += vector_length) {
@@ -89,9 +89,12 @@ NK_PUBLIC void nk_angular_f16_rvvhalf(nk_f16_t const *a_scalars, nk_f16_t const
 
     // Single horizontal reduction after the loop
     vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
-    nk_f32_t dot = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m2_f32m1(dot_f32m2, zero_f32m1, vlmax));
-    nk_f32_t a_sq = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m2_f32m1(a_sq_f32m2, zero_f32m1, vlmax));
-    nk_f32_t b_sq = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m2_f32m1(b_sq_f32m2, zero_f32m1, vlmax));
+    nk_f32_t dot = __riscv_vfmv_f_s_f32m1_f32(
+        __riscv_vfredusum_vs_f32m2_f32m1(dot_f32m2, zero_f32m1, max_vector_length));
+    nk_f32_t a_sq = __riscv_vfmv_f_s_f32m1_f32(
+        __riscv_vfredusum_vs_f32m2_f32m1(a_sq_f32m2, zero_f32m1, max_vector_length));
+    nk_f32_t b_sq = __riscv_vfmv_f_s_f32m1_f32(
+        __riscv_vfredusum_vs_f32m2_f32m1(b_sq_f32m2, zero_f32m1, max_vector_length));
 
     // Normalize: 1 − dot / sqrt(‖a‖² × ‖b‖²)
     if (a_sq == 0.0f && b_sq == 0.0f) { *result = 0.0f; }

package/include/numkong/spatial/serial.h

@@ -40,11 +40,11 @@ extern "C" {
 #define nk_define_sqeuclidean_(input_type, accumulator_type, output_type, load_and_convert)                         \
     NK_PUBLIC void nk_sqeuclidean_##input_type##_serial(nk_##input_type##_t const *a, nk_##input_type##_t const *b, \
                                                         nk_size_t n, nk_##output_type##_t *result) {                \
-        nk_##accumulator_type##_t sum = 0, compensation = 0, a_element, b_element;                                  \
+        nk_##accumulator_type##_t sum = 0, compensation = 0, a_value, b_value;                                      \
         for (nk_size_t i = 0; i != n; ++i) {                                                                        \
-            load_and_convert(a + i, &a_element);                                                                    \
-            load_and_convert(b + i, &b_element);                                                                    \
-            nk_##accumulator_type##_t diff = a_element - b_element;                                                 \
+            load_and_convert(a + i, &a_value);                                                                      \
+            load_and_convert(b + i, &b_value);                                                                      \
+            nk_##accumulator_type##_t diff = a_value - b_value;                                                     \
             nk_##accumulator_type##_t term = diff * diff, t = sum + term;                                           \
             compensation += (nk_##accumulator_type##_abs_(sum) >= nk_##accumulator_type##_abs_(term))               \
                                 ? ((sum - t) + term)                                                                \
@@ -74,14 +74,14 @@ extern "C" {
 #define nk_define_angular_(input_type, accumulator_type, output_type, load_and_convert, compute_rsqrt)            \
     NK_PUBLIC void nk_angular_##input_type##_serial(nk_##input_type##_t const *a, nk_##input_type##_t const *b,   \
                                                     nk_size_t n, nk_##output_type##_t *result) {                  \
-        nk_##accumulator_type##_t dot_sum = 0, a_sum = 0, b_sum = 0, a_element, b_element;                        \
+        nk_##accumulator_type##_t dot_sum = 0, a_sum = 0, b_sum = 0, a_value, b_value;                            \
         nk_##accumulator_type##_t compensation_dot = 0, compensation_a = 0, compensation_b = 0;                   \
         for (nk_size_t i = 0; i != n; ++i) {                                                                      \
-            load_and_convert(a + i, &a_element);                                                                  \
-            load_and_convert(b + i, &b_element);                                                                  \
-            nk_##accumulator_type##_t term_dot = a_element * b_element, t_dot = dot_sum + term_dot;               \
-            nk_##accumulator_type##_t term_a = a_element * a_element, t_a = a_sum + term_a;                       \
-            nk_##accumulator_type##_t term_b = b_element * b_element, t_b = b_sum + term_b;                       \
+            load_and_convert(a + i, &a_value);                                                                    \
+            load_and_convert(b + i, &b_value);                                                                    \
+            nk_##accumulator_type##_t term_dot = a_value * b_value, t_dot = dot_sum + term_dot;                   \
+            nk_##accumulator_type##_t term_a = a_value * a_value, t_a = a_sum + term_a;                           \
+            nk_##accumulator_type##_t term_b = b_value * b_value, t_b = b_sum + term_b;                           \
             compensation_dot += (nk_##accumulator_type##_abs_(dot_sum) >= nk_##accumulator_type##_abs_(term_dot)) \
                                     ? ((dot_sum - t_dot) + term_dot)                                              \
                                     : ((term_dot - t_dot) + dot_sum);                                             \
@@ -101,8 +101,9 @@ extern "C" {
         if (a_norm_sq == 0 && b_norm_sq == 0) { *result = 0; }                                                    \
         else if (dot_product == 0) { *result = 1; }                                                               \
         else {                                                                                                    \
-            nk_##output_type##_t unclipped_distance = 1 - dot_product * compute_rsqrt(a_norm_sq) *                \
-                                                              compute_rsqrt(b_norm_sq);                           \
+            nk_##output_type##_t unclipped_distance = (nk_##output_type##_t)(                                     \
+                1 - (nk_##output_type##_t)dot_product * compute_rsqrt((nk_##output_type##_t)a_norm_sq) *          \
+                        compute_rsqrt((nk_##output_type##_t)b_norm_sq));                                          \
             *result = unclipped_distance > 0 ? unclipped_distance : 0;                                            \
         }                                                                                                         \
     }

package/include/numkong/spatial/sierra.h

@@ -8,12 +8,12 @@
  *
  *  @section spatial_sierra_instructions AVXVNNIINT8 Instructions Performance
  *
- *      Intrinsic                   Instruction                     Sierra Forest
- *      _mm256_dpbssds_epi32        VPDPBSSDS (YMM, YMM, YMM)       4cy @ p05
- *      _mm256_dpbssd_epi32         VPDPBSSD (YMM, YMM, YMM)        4cy @ p05
- *      _mm256_dpbuud_epi32         VPDPBUUD (YMM, YMM, YMM)        4cy @ p05
- *      _mm_rsqrt_ps                VRSQRTPS (XMM, XMM)             5cy @ p0
- *      _mm_sqrt_ss                 VSQRTSS (XMM, XMM, XMM)        12cy @ p0
+ *      Intrinsic             Instruction                Sierra Forest
+ *      _mm256_dpbssds_epi32  VPDPBSSDS (YMM, YMM, YMM)  4cy @ p05
+ *      _mm256_dpbssd_epi32   VPDPBSSD (YMM, YMM, YMM)   4cy @ p05
+ *      _mm256_dpbuud_epi32   VPDPBUUD (YMM, YMM, YMM)   4cy @ p05
+ *      _mm_rsqrt_ps          VRSQRTPS (XMM, XMM)        5cy @ p0
+ *      _mm_sqrt_ss           VSQRTSS (XMM, XMM, XMM)    12cy @ p0
  *
  *  Sierra Forest (AVXVNNIINT8) provides native signed x signed and unsigned x unsigned
  *  dot products, eliminating the need for algebraic corrections required on Alder Lake.
@@ -67,7 +67,8 @@ NK_PUBLIC void nk_angular_i8_sierra(nk_i8_t const *a, nk_i8_t const *b, nk_size_
         b_norm_sq_i32 += b_element_i32 * b_element_i32;
     }
 
-    *result = nk_angular_normalize_f32_haswell_(dot_product_i32, a_norm_sq_i32, b_norm_sq_i32);
+    *result = nk_angular_normalize_f32_haswell_((nk_f32_t)dot_product_i32, (nk_f32_t)a_norm_sq_i32,
+                                                (nk_f32_t)b_norm_sq_i32);
 }
 
 NK_PUBLIC void nk_sqeuclidean_i8_sierra(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_u32_t *result) {
@@ -132,7 +133,8 @@ NK_PUBLIC void nk_angular_u8_sierra(nk_u8_t const *a, nk_u8_t const *b, nk_size_
         b_norm_sq_i32 += b_element_i32 * b_element_i32;
     }
 
-    *result = nk_angular_normalize_f32_haswell_(dot_product_i32, a_norm_sq_i32, b_norm_sq_i32);
+    *result = nk_angular_normalize_f32_haswell_((nk_f32_t)dot_product_i32, (nk_f32_t)a_norm_sq_i32,
+                                                (nk_f32_t)b_norm_sq_i32);
 }
 
 NK_PUBLIC void nk_sqeuclidean_u8_sierra(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_u32_t *result) {
@@ -177,15 +179,15 @@ NK_PUBLIC void nk_angular_e2m3_sierra(nk_e2m3_t const *a_scalars, nk_e2m3_t cons
     // Every e2m3 value × 16 is an exact integer in [-120, +120].
     // DPBSSD(signed, signed) eliminates the need for unsigned conversion tricks.
     //
-    __m256i const lut_lower_u8x32 = _mm256_set_epi8(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, 30, 28,
-                                                    26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0);
-    __m256i const lut_upper_u8x32 = _mm256_set_epi8(120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32,
-                                                    120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32);
+    __m256i const lut_low_u8x32 = _mm256_set_epi8(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, 30, 28, 26,
+                                                  24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0);
+    __m256i const lut_high_u8x32 = _mm256_set_epi8(120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32,
+                                                   120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32);
     __m256i const nibble_mask_u8x32 = _mm256_set1_epi8(0x0F);
     __m256i const magnitude_mask_u8x32 = _mm256_set1_epi8(0x1F);
     __m256i const half_select_u8x32 = _mm256_set1_epi8(0x10);
     __m256i const sign_mask_u8x32 = _mm256_set1_epi8(0x20);
-    __m256i dot_i32x8 = _mm256_setzero_si256();
+    __m256i ab_i32x8 = _mm256_setzero_si256();
     __m256i a_norm_i32x8 = _mm256_setzero_si256();
     __m256i b_norm_i32x8 = _mm256_setzero_si256();
     __m256i a_e2m3_u8x32, b_e2m3_u8x32;
@@ -207,35 +209,39 @@ nk_angular_e2m3_sierra_cycle:
 
     // Decode a: extract magnitude, dual-VPSHUFB LUT, apply sign
     __m256i a_magnitude_u8x32 = _mm256_and_si256(a_e2m3_u8x32, magnitude_mask_u8x32);
-    __m256i a_shuffle_idx = _mm256_and_si256(a_magnitude_u8x32, nibble_mask_u8x32);
-    __m256i a_upper_sel = _mm256_cmpeq_epi8(_mm256_and_si256(a_magnitude_u8x32, half_select_u8x32), half_select_u8x32);
-    __m256i a_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_lower_u8x32, a_shuffle_idx),
-                                                  _mm256_shuffle_epi8(lut_upper_u8x32, a_shuffle_idx), a_upper_sel);
+    __m256i a_shuffle_index_u8x32 = _mm256_and_si256(a_magnitude_u8x32, nibble_mask_u8x32);
+    __m256i a_high_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(a_magnitude_u8x32, half_select_u8x32),
+                                                    half_select_u8x32);
+    __m256i a_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_low_u8x32, a_shuffle_index_u8x32),
+                                                  _mm256_shuffle_epi8(lut_high_u8x32, a_shuffle_index_u8x32),
+                                                  a_high_select_u8x32);
     __m256i a_negate = _mm256_cmpeq_epi8(_mm256_and_si256(a_e2m3_u8x32, sign_mask_u8x32), sign_mask_u8x32);
     __m256i a_signed_i8x32 = _mm256_blendv_epi8(a_unsigned_u8x32,
                                                 _mm256_sub_epi8(_mm256_setzero_si256(), a_unsigned_u8x32), a_negate);
 
     // Decode b: same LUT decode + sign
     __m256i b_magnitude_u8x32 = _mm256_and_si256(b_e2m3_u8x32, magnitude_mask_u8x32);
-    __m256i b_shuffle_idx = _mm256_and_si256(b_magnitude_u8x32, nibble_mask_u8x32);
-    __m256i b_upper_sel = _mm256_cmpeq_epi8(_mm256_and_si256(b_magnitude_u8x32, half_select_u8x32), half_select_u8x32);
-    __m256i b_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_lower_u8x32, b_shuffle_idx),
-                                                  _mm256_shuffle_epi8(lut_upper_u8x32, b_shuffle_idx), b_upper_sel);
+    __m256i b_shuffle_index_u8x32 = _mm256_and_si256(b_magnitude_u8x32, nibble_mask_u8x32);
+    __m256i b_high_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(b_magnitude_u8x32, half_select_u8x32),
+                                                    half_select_u8x32);
+    __m256i b_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_low_u8x32, b_shuffle_index_u8x32),
+                                                  _mm256_shuffle_epi8(lut_high_u8x32, b_shuffle_index_u8x32),
+                                                  b_high_select_u8x32);
     __m256i b_negate = _mm256_cmpeq_epi8(_mm256_and_si256(b_e2m3_u8x32, sign_mask_u8x32), sign_mask_u8x32);
     __m256i b_signed_i8x32 = _mm256_blendv_epi8(b_unsigned_u8x32,
                                                 _mm256_sub_epi8(_mm256_setzero_si256(), b_unsigned_u8x32), b_negate);
 
     // VPDPBSSD: signed × signed → i32
-    dot_i32x8 = _mm256_dpbssd_epi32(dot_i32x8, a_signed_i8x32, b_signed_i8x32);
+    ab_i32x8 = _mm256_dpbssd_epi32(ab_i32x8, a_signed_i8x32, b_signed_i8x32);
     a_norm_i32x8 = _mm256_dpbssd_epi32(a_norm_i32x8, a_signed_i8x32, a_signed_i8x32);
     b_norm_i32x8 = _mm256_dpbssd_epi32(b_norm_i32x8, b_signed_i8x32, b_signed_i8x32);
 
     if (count_scalars) goto nk_angular_e2m3_sierra_cycle;
 
-    nk_i32_t dot_i32 = nk_reduce_add_i32x8_haswell_(dot_i32x8);
+    nk_i32_t dot_i32 = nk_reduce_add_i32x8_haswell_(ab_i32x8);
     nk_i32_t a_norm_i32 = nk_reduce_add_i32x8_haswell_(a_norm_i32x8);
     nk_i32_t b_norm_i32 = nk_reduce_add_i32x8_haswell_(b_norm_i32x8);
-    *result = nk_angular_normalize_f32_haswell_(dot_i32, a_norm_i32, b_norm_i32);
+    *result = nk_angular_normalize_f32_haswell_((nk_f32_t)dot_i32, (nk_f32_t)a_norm_i32, (nk_f32_t)b_norm_i32);
 }
 
 NK_PUBLIC void nk_sqeuclidean_e2m3_sierra(nk_e2m3_t const *a_scalars, nk_e2m3_t const *b_scalars,
@@ -243,15 +249,15 @@ NK_PUBLIC void nk_sqeuclidean_e2m3_sierra(nk_e2m3_t const *a_scalars, nk_e2m3_t
     // Squared Euclidean distance for e2m3 using norm decomposition + VPDPBSSD.
     // ||a-b||^2 = ||a||^2 + ||b||^2 - 2*dot(a,b)
     //
-    __m256i const lut_lower_u8x32 = _mm256_set_epi8(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, 30, 28,
-                                                    26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0);
-    __m256i const lut_upper_u8x32 = _mm256_set_epi8(120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32,
-                                                    120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32);
+    __m256i const lut_low_u8x32 = _mm256_set_epi8(30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, 30, 28, 26,
+                                                  24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0);
+    __m256i const lut_high_u8x32 = _mm256_set_epi8(120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32,
+                                                   120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32);
     __m256i const nibble_mask_u8x32 = _mm256_set1_epi8(0x0F);
     __m256i const magnitude_mask_u8x32 = _mm256_set1_epi8(0x1F);
     __m256i const half_select_u8x32 = _mm256_set1_epi8(0x10);
     __m256i const sign_mask_u8x32 = _mm256_set1_epi8(0x20);
-    __m256i dot_i32x8 = _mm256_setzero_si256();
+    __m256i ab_i32x8 = _mm256_setzero_si256();
     __m256i a_norm_i32x8 = _mm256_setzero_si256();
     __m256i b_norm_i32x8 = _mm256_setzero_si256();
     __m256i a_e2m3_u8x32, b_e2m3_u8x32;
@@ -273,31 +279,35 @@ nk_sqeuclidean_e2m3_sierra_cycle:
 
     // Decode a
     __m256i a_magnitude_u8x32 = _mm256_and_si256(a_e2m3_u8x32, magnitude_mask_u8x32);
-    __m256i a_shuffle_idx = _mm256_and_si256(a_magnitude_u8x32, nibble_mask_u8x32);
-    __m256i a_upper_sel = _mm256_cmpeq_epi8(_mm256_and_si256(a_magnitude_u8x32, half_select_u8x32), half_select_u8x32);
-    __m256i a_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_lower_u8x32, a_shuffle_idx),
-                                                  _mm256_shuffle_epi8(lut_upper_u8x32, a_shuffle_idx), a_upper_sel);
+    __m256i a_shuffle_index_u8x32 = _mm256_and_si256(a_magnitude_u8x32, nibble_mask_u8x32);
+    __m256i a_high_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(a_magnitude_u8x32, half_select_u8x32),
+                                                    half_select_u8x32);
+    __m256i a_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_low_u8x32, a_shuffle_index_u8x32),
+                                                  _mm256_shuffle_epi8(lut_high_u8x32, a_shuffle_index_u8x32),
+                                                  a_high_select_u8x32);
     __m256i a_negate = _mm256_cmpeq_epi8(_mm256_and_si256(a_e2m3_u8x32, sign_mask_u8x32), sign_mask_u8x32);
     __m256i a_signed_i8x32 = _mm256_blendv_epi8(a_unsigned_u8x32,
                                                 _mm256_sub_epi8(_mm256_setzero_si256(), a_unsigned_u8x32), a_negate);
 
     // Decode b
     __m256i b_magnitude_u8x32 = _mm256_and_si256(b_e2m3_u8x32, magnitude_mask_u8x32);
-    __m256i b_shuffle_idx = _mm256_and_si256(b_magnitude_u8x32, nibble_mask_u8x32);
-    __m256i b_upper_sel = _mm256_cmpeq_epi8(_mm256_and_si256(b_magnitude_u8x32, half_select_u8x32), half_select_u8x32);
-    __m256i b_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_lower_u8x32, b_shuffle_idx),
-                                                  _mm256_shuffle_epi8(lut_upper_u8x32, b_shuffle_idx), b_upper_sel);
+    __m256i b_shuffle_index_u8x32 = _mm256_and_si256(b_magnitude_u8x32, nibble_mask_u8x32);
+    __m256i b_high_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(b_magnitude_u8x32, half_select_u8x32),
+                                                    half_select_u8x32);
+    __m256i b_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_low_u8x32, b_shuffle_index_u8x32),
+                                                  _mm256_shuffle_epi8(lut_high_u8x32, b_shuffle_index_u8x32),
+                                                  b_high_select_u8x32);
     __m256i b_negate = _mm256_cmpeq_epi8(_mm256_and_si256(b_e2m3_u8x32, sign_mask_u8x32), sign_mask_u8x32);
     __m256i b_signed_i8x32 = _mm256_blendv_epi8(b_unsigned_u8x32,
                                                 _mm256_sub_epi8(_mm256_setzero_si256(), b_unsigned_u8x32), b_negate);
 
-    dot_i32x8 = _mm256_dpbssd_epi32(dot_i32x8, a_signed_i8x32, b_signed_i8x32);
+    ab_i32x8 = _mm256_dpbssd_epi32(ab_i32x8, a_signed_i8x32, b_signed_i8x32);
     a_norm_i32x8 = _mm256_dpbssd_epi32(a_norm_i32x8, a_signed_i8x32, a_signed_i8x32);
     b_norm_i32x8 = _mm256_dpbssd_epi32(b_norm_i32x8, b_signed_i8x32, b_signed_i8x32);
 
     if (count_scalars) goto nk_sqeuclidean_e2m3_sierra_cycle;
 
-    nk_i32_t dot_i32 = nk_reduce_add_i32x8_haswell_(dot_i32x8);
+    nk_i32_t dot_i32 = nk_reduce_add_i32x8_haswell_(ab_i32x8);
     nk_i32_t a_norm_i32 = nk_reduce_add_i32x8_haswell_(a_norm_i32x8);
     nk_i32_t b_norm_i32 = nk_reduce_add_i32x8_haswell_(b_norm_i32x8);
     *result = (nk_f32_t)(a_norm_i32 + b_norm_i32 - 2 * dot_i32) / 256.0f;
@@ -308,6 +318,189 @@ NK_PUBLIC void nk_euclidean_e2m3_sierra(nk_e2m3_t const *a, nk_e2m3_t const *b,
     *result = nk_f32_sqrt_haswell(*result);
 }
 
+NK_PUBLIC void nk_sqeuclidean_e3m2_sierra(nk_e3m2_t const *a_scalars, nk_e3m2_t const *b_scalars,
+                                          nk_size_t count_scalars, nk_f32_t *result) {
+    // E3M2 squared Euclidean distance via direct difference squaring.
+    __m256i const lut_low_byte_first_u8x32 = _mm256_set_epi8(  //
+        28, 24, 20, 16, 14, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1, 0, //
+        28, 24, 20, 16, 14, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1, 0);
+    __m256i const lut_low_byte_second_u8x32 = _mm256_set_epi8(                                                    //
+        (char)192, (char)128, 64, 0, (char)224, (char)192, (char)160, (char)128, 112, 96, 80, 64, 56, 48, 40, 32, //
+        (char)192, (char)128, 64, 0, (char)224, (char)192, (char)160, (char)128, 112, 96, 80, 64, 56, 48, 40, 32);
+    __m256i const nibble_mask_u8x32 = _mm256_set1_epi8(0x0F);
+    __m256i const magnitude_mask_u8x32 = _mm256_set1_epi8(0x1F);
+    __m256i const half_select_u8x32 = _mm256_set1_epi8(0x10);
+    __m256i const sign_mask_u8x32 = _mm256_set1_epi8(0x20);
+    __m256i const high_threshold_u8x32 = _mm256_set1_epi8(27);
+    __m256i const ones_u8x32 = _mm256_set1_epi8(1);
+    __m256i const ones_i16x16 = _mm256_set1_epi16(1);
+    __m256i sum_i32x8 = _mm256_setzero_si256();
+    __m256i a_e3m2_u8x32, b_e3m2_u8x32;
+
+nk_sqeuclidean_e3m2_sierra_cycle:
+    if (count_scalars < 32) {
+        nk_b256_vec_t a_vec, b_vec;
+        nk_partial_load_b8x32_serial_(a_scalars, &a_vec, count_scalars);
+        nk_partial_load_b8x32_serial_(b_scalars, &b_vec, count_scalars);
+        a_e3m2_u8x32 = a_vec.ymm;
+        b_e3m2_u8x32 = b_vec.ymm;
+        count_scalars = 0;
+    }
+    else {
+        a_e3m2_u8x32 = _mm256_loadu_si256((__m256i const *)a_scalars);
+        b_e3m2_u8x32 = _mm256_loadu_si256((__m256i const *)b_scalars);
+        a_scalars += 32, b_scalars += 32, count_scalars -= 32;
+    }
+
+    // Decode both to unsigned i16 via dual-VPSHUFB + interleave
+    __m256i a_magnitude_u8x32 = _mm256_and_si256(a_e3m2_u8x32, magnitude_mask_u8x32);
+    __m256i b_magnitude_u8x32 = _mm256_and_si256(b_e3m2_u8x32, magnitude_mask_u8x32);
+    __m256i a_shuffle_index_u8x32 = _mm256_and_si256(a_magnitude_u8x32, nibble_mask_u8x32);
+    __m256i b_shuffle_index_u8x32 = _mm256_and_si256(b_magnitude_u8x32, nibble_mask_u8x32);
+    __m256i a_high_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(a_magnitude_u8x32, half_select_u8x32),
+                                                    half_select_u8x32);
+    __m256i b_high_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(b_magnitude_u8x32, half_select_u8x32),
+                                                    half_select_u8x32);
+    __m256i a_low_bytes_u8x32 = _mm256_blendv_epi8(
+        _mm256_shuffle_epi8(lut_low_byte_first_u8x32, a_shuffle_index_u8x32),
+        _mm256_shuffle_epi8(lut_low_byte_second_u8x32, a_shuffle_index_u8x32), a_high_select_u8x32);
+    __m256i b_low_bytes_u8x32 = _mm256_blendv_epi8(
+        _mm256_shuffle_epi8(lut_low_byte_first_u8x32, b_shuffle_index_u8x32),
+        _mm256_shuffle_epi8(lut_low_byte_second_u8x32, b_shuffle_index_u8x32), b_high_select_u8x32);
+    __m256i a_high_bytes_u8x32 = _mm256_and_si256(_mm256_cmpgt_epi8(a_magnitude_u8x32, high_threshold_u8x32),
+                                                  ones_u8x32);
+    __m256i b_high_bytes_u8x32 = _mm256_and_si256(_mm256_cmpgt_epi8(b_magnitude_u8x32, high_threshold_u8x32),
+                                                  ones_u8x32);
+
+    // Interleave to i16 and apply signs
+    __m256i a_low_i16x16 = _mm256_unpacklo_epi8(a_low_bytes_u8x32, a_high_bytes_u8x32);
+    __m256i a_high_i16x16 = _mm256_unpackhi_epi8(a_low_bytes_u8x32, a_high_bytes_u8x32);
+    __m256i b_low_i16x16 = _mm256_unpacklo_epi8(b_low_bytes_u8x32, b_high_bytes_u8x32);
+    __m256i b_high_i16x16 = _mm256_unpackhi_epi8(b_low_bytes_u8x32, b_high_bytes_u8x32);
+
+    __m256i a_negative_mask_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(a_e3m2_u8x32, sign_mask_u8x32), sign_mask_u8x32);
+    __m256i b_negative_mask_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(b_e3m2_u8x32, sign_mask_u8x32), sign_mask_u8x32);
+    __m256i a_sign_low_i16x16 = _mm256_or_si256(_mm256_unpacklo_epi8(a_negative_mask_u8x32, a_negative_mask_u8x32),
+                                                ones_i16x16);
+    __m256i a_sign_high_i16x16 = _mm256_or_si256(_mm256_unpackhi_epi8(a_negative_mask_u8x32, a_negative_mask_u8x32),
+                                                 ones_i16x16);
+    __m256i b_sign_low_i16x16 = _mm256_or_si256(_mm256_unpacklo_epi8(b_negative_mask_u8x32, b_negative_mask_u8x32),
+                                                ones_i16x16);
+    __m256i b_sign_high_i16x16 = _mm256_or_si256(_mm256_unpackhi_epi8(b_negative_mask_u8x32, b_negative_mask_u8x32),
+                                                 ones_i16x16);
+    __m256i a_signed_low_i16x16 = _mm256_sign_epi16(a_low_i16x16, a_sign_low_i16x16);
+    __m256i a_signed_high_i16x16 = _mm256_sign_epi16(a_high_i16x16, a_sign_high_i16x16);
+    __m256i b_signed_low_i16x16 = _mm256_sign_epi16(b_low_i16x16, b_sign_low_i16x16);
+    __m256i b_signed_high_i16x16 = _mm256_sign_epi16(b_high_i16x16, b_sign_high_i16x16);
+
+    // Direct difference squaring: (a-b)² via VPMADDWD
+    __m256i diff_low_i16x16 = _mm256_sub_epi16(a_signed_low_i16x16, b_signed_low_i16x16);
+    __m256i diff_high_i16x16 = _mm256_sub_epi16(a_signed_high_i16x16, b_signed_high_i16x16);
+    sum_i32x8 = _mm256_add_epi32(sum_i32x8, _mm256_madd_epi16(diff_low_i16x16, diff_low_i16x16));
+    sum_i32x8 = _mm256_add_epi32(sum_i32x8, _mm256_madd_epi16(diff_high_i16x16, diff_high_i16x16));
+
+    if (count_scalars) goto nk_sqeuclidean_e3m2_sierra_cycle;
+    *result = (nk_f32_t)nk_reduce_add_i32x8_haswell_(sum_i32x8) / 256.0f;
+}
+
+NK_PUBLIC void nk_euclidean_e3m2_sierra(nk_e3m2_t const *a, nk_e3m2_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_sqeuclidean_e3m2_sierra(a, b, n, result);
+    *result = nk_f32_sqrt_haswell(*result);
+}
+
+NK_PUBLIC void nk_angular_e3m2_sierra(nk_e3m2_t const *a_scalars, nk_e3m2_t const *b_scalars, nk_size_t count_scalars,
+                                      nk_f32_t *result) {
+    // E3M2 angular distance via VPMADDWD integer MAC.
+    __m256i const lut_low_byte_first_u8x32 = _mm256_set_epi8(  //
+        28, 24, 20, 16, 14, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1, 0, //
+        28, 24, 20, 16, 14, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1, 0);
+    __m256i const lut_low_byte_second_u8x32 = _mm256_set_epi8(                                                    //
+        (char)192, (char)128, 64, 0, (char)224, (char)192, (char)160, (char)128, 112, 96, 80, 64, 56, 48, 40, 32, //
+        (char)192, (char)128, 64, 0, (char)224, (char)192, (char)160, (char)128, 112, 96, 80, 64, 56, 48, 40, 32);
+    __m256i const nibble_mask_u8x32 = _mm256_set1_epi8(0x0F);
+    __m256i const magnitude_mask_u8x32 = _mm256_set1_epi8(0x1F);
+    __m256i const half_select_u8x32 = _mm256_set1_epi8(0x10);
+    __m256i const sign_mask_u8x32 = _mm256_set1_epi8(0x20);
+    __m256i const high_threshold_u8x32 = _mm256_set1_epi8(27);
+    __m256i const ones_u8x32 = _mm256_set1_epi8(1);
+    __m256i const ones_i16x16 = _mm256_set1_epi16(1);
+    __m256i ab_i32x8 = _mm256_setzero_si256();
+    __m256i a_norm_i32x8 = _mm256_setzero_si256();
+    __m256i b_norm_i32x8 = _mm256_setzero_si256();
+    __m256i a_e3m2_u8x32, b_e3m2_u8x32;
+
+nk_angular_e3m2_sierra_cycle:
+    if (count_scalars < 32) {
+        nk_b256_vec_t a_vec, b_vec;
+        nk_partial_load_b8x32_serial_(a_scalars, &a_vec, count_scalars);
+        nk_partial_load_b8x32_serial_(b_scalars, &b_vec, count_scalars);
+        a_e3m2_u8x32 = a_vec.ymm;
+        b_e3m2_u8x32 = b_vec.ymm;
+        count_scalars = 0;
+    }
+    else {
+        a_e3m2_u8x32 = _mm256_loadu_si256((__m256i const *)a_scalars);
+        b_e3m2_u8x32 = _mm256_loadu_si256((__m256i const *)b_scalars);
+        a_scalars += 32, b_scalars += 32, count_scalars -= 32;
+    }
+
+    // Decode both to unsigned i16
+    __m256i a_magnitude_u8x32 = _mm256_and_si256(a_e3m2_u8x32, magnitude_mask_u8x32);
+    __m256i b_magnitude_u8x32 = _mm256_and_si256(b_e3m2_u8x32, magnitude_mask_u8x32);
+    __m256i a_shuffle_index_u8x32 = _mm256_and_si256(a_magnitude_u8x32, nibble_mask_u8x32);
+    __m256i b_shuffle_index_u8x32 = _mm256_and_si256(b_magnitude_u8x32, nibble_mask_u8x32);
+    __m256i a_high_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(a_magnitude_u8x32, half_select_u8x32),
+                                                    half_select_u8x32);
+    __m256i b_high_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(b_magnitude_u8x32, half_select_u8x32),
+                                                    half_select_u8x32);
+    __m256i a_low_bytes_u8x32 = _mm256_blendv_epi8(
+        _mm256_shuffle_epi8(lut_low_byte_first_u8x32, a_shuffle_index_u8x32),
+        _mm256_shuffle_epi8(lut_low_byte_second_u8x32, a_shuffle_index_u8x32), a_high_select_u8x32);
+    __m256i b_low_bytes_u8x32 = _mm256_blendv_epi8(
+        _mm256_shuffle_epi8(lut_low_byte_first_u8x32, b_shuffle_index_u8x32),
+        _mm256_shuffle_epi8(lut_low_byte_second_u8x32, b_shuffle_index_u8x32), b_high_select_u8x32);
+    __m256i a_high_bytes_u8x32 = _mm256_and_si256(_mm256_cmpgt_epi8(a_magnitude_u8x32, high_threshold_u8x32),
+                                                  ones_u8x32);
+    __m256i b_high_bytes_u8x32 = _mm256_and_si256(_mm256_cmpgt_epi8(b_magnitude_u8x32, high_threshold_u8x32),
+                                                  ones_u8x32);
+    __m256i a_low_i16x16 = _mm256_unpacklo_epi8(a_low_bytes_u8x32, a_high_bytes_u8x32);
+    __m256i a_high_i16x16 = _mm256_unpackhi_epi8(a_low_bytes_u8x32, a_high_bytes_u8x32);
+    __m256i b_low_i16x16 = _mm256_unpacklo_epi8(b_low_bytes_u8x32, b_high_bytes_u8x32);
+    __m256i b_high_i16x16 = _mm256_unpackhi_epi8(b_low_bytes_u8x32, b_high_bytes_u8x32);
+
+    // Apply signs individually
+    __m256i a_negative_mask_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(a_e3m2_u8x32, sign_mask_u8x32), sign_mask_u8x32);
+    __m256i b_negative_mask_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(b_e3m2_u8x32, sign_mask_u8x32), sign_mask_u8x32);
+    __m256i a_sign_low_i16x16 = _mm256_or_si256(_mm256_unpacklo_epi8(a_negative_mask_u8x32, a_negative_mask_u8x32),
+                                                ones_i16x16);
+    __m256i a_sign_high_i16x16 = _mm256_or_si256(_mm256_unpackhi_epi8(a_negative_mask_u8x32, a_negative_mask_u8x32),
+                                                 ones_i16x16);
+    __m256i b_sign_low_i16x16 = _mm256_or_si256(_mm256_unpacklo_epi8(b_negative_mask_u8x32, b_negative_mask_u8x32),
+                                                ones_i16x16);
+    __m256i b_sign_high_i16x16 = _mm256_or_si256(_mm256_unpackhi_epi8(b_negative_mask_u8x32, b_negative_mask_u8x32),
+                                                 ones_i16x16);
+    __m256i a_signed_low_i16x16 = _mm256_sign_epi16(a_low_i16x16, a_sign_low_i16x16);
+    __m256i a_signed_high_i16x16 = _mm256_sign_epi16(a_high_i16x16, a_sign_high_i16x16);
+    __m256i b_signed_low_i16x16 = _mm256_sign_epi16(b_low_i16x16, b_sign_low_i16x16);
+    __m256i b_signed_high_i16x16 = _mm256_sign_epi16(b_high_i16x16, b_sign_high_i16x16);
+
+    // dot(a,b) + a² + b² via VPMADDWD
+    ab_i32x8 = _mm256_add_epi32(ab_i32x8, _mm256_madd_epi16(a_signed_low_i16x16, b_signed_low_i16x16));
+    ab_i32x8 = _mm256_add_epi32(ab_i32x8, _mm256_madd_epi16(a_signed_high_i16x16, b_signed_high_i16x16));
+    a_norm_i32x8 = _mm256_add_epi32(a_norm_i32x8, _mm256_madd_epi16(a_low_i16x16, a_low_i16x16));
+    a_norm_i32x8 = _mm256_add_epi32(a_norm_i32x8, _mm256_madd_epi16(a_high_i16x16, a_high_i16x16));
+    b_norm_i32x8 = _mm256_add_epi32(b_norm_i32x8, _mm256_madd_epi16(b_low_i16x16, b_low_i16x16));
+    b_norm_i32x8 = _mm256_add_epi32(b_norm_i32x8, _mm256_madd_epi16(b_high_i16x16, b_high_i16x16));
+
+    if (count_scalars) goto nk_angular_e3m2_sierra_cycle;
+
+    nk_i32_t dot_i32 = nk_reduce_add_i32x8_haswell_(ab_i32x8);
+    nk_i32_t a_norm_i32 = nk_reduce_add_i32x8_haswell_(a_norm_i32x8);
+    nk_i32_t b_norm_i32 = nk_reduce_add_i32x8_haswell_(b_norm_i32x8);
+    *result = nk_angular_normalize_f32_haswell_((nk_f32_t)dot_i32 / 256.0f, (nk_f32_t)a_norm_i32 / 256.0f,
+                                                (nk_f32_t)b_norm_i32 / 256.0f);
+}
+
 #if defined(__clang__)
 #pragma clang attribute pop
 #elif defined(__GNUC__)