numkong 7.4.5 → 7.6.0

package/include/numkong/cast/skylake.h

@@ -175,60 +175,63 @@ NK_INTERNAL __m256i nk_f32x16_to_bf16x16_skylake_(__m512 a) {
     return _mm512_cvtepi32_epi16(x);
 }
 
-/** @brief Convert 16x e4m3 → 16x f32 via bit manipulation (AVX-512).
- *  E4M3 format: S EEEE MMM (bias=7). F32: sign<<31, (exp+120)<<23, mantissa<<20.
- *  Subnormals (exp=0): value = mantissa × 2⁽¹⁻⁷⁾ × 2⁻³ = mantissa ÷ 512. */
+/** @brief Convert 16x e4m3 → 16x f32 via Giesen-style fake-F16 cast (AVX-512 + F16C).
+ *  E4M3 `byte = S EEEE MMM` (bias 7). Shifting the magnitude into F16 positions
+ *  `((byte & 0x7F) << 7) | ((byte & 0x80) << 8)` yields a fake F16 whose F16 value
+ *  differs from the true E4M3 magnitude by exactly 2⁸ (bias delta 15 − 7). The
+ *  fake F16 is widened via `vcvtph2ps` and corrected by ×256 in F32. Subnormal
+ *  handling falls out for free via F16 subnormal semantics. NaN (|byte|==0x7F) is
+ *  the sole E4M3 special value that would misinterpret as finite; blended
+ *  explicitly with F32 quiet NaN bits. */
 NK_INTERNAL __m512 nk_e4m3x16_to_f32x16_skylake_(__m128i e4m3_i8x16) {
-    __m512i e4m3_i32x16 = _mm512_cvtepu8_epi32(e4m3_i8x16);
-
-    // Extract fields
-    __m512i exp_i32x16 = _mm512_and_si512(_mm512_srli_epi32(e4m3_i32x16, 3), _mm512_set1_epi32(0x0F));
-    __m512i mantissa_i32x16 = _mm512_and_si512(e4m3_i32x16, _mm512_set1_epi32(0x07));
-    __m512i sign_i32x16 = _mm512_slli_epi32(_mm512_srli_epi32(e4m3_i32x16, 7), 31);
-
-    // Normal path: sign | ((exp+120)<<23) | (mantissa<<20)
-    __m512i f32_exp_i32x16 = _mm512_slli_epi32(_mm512_add_epi32(exp_i32x16, _mm512_set1_epi32(120)), 23);
-    __m512i f32_mantissa_i32x16 = _mm512_slli_epi32(mantissa_i32x16, 20);
-    __m512 result_f32x16 = _mm512_castsi512_ps(
-        _mm512_ternarylogic_epi32(sign_i32x16, f32_exp_i32x16, f32_mantissa_i32x16, 0xFE));
-
-    // Subnormal fix: vpermps from 8-entry LUT (repeated to fill 16 lanes)
-    __mmask16 is_subnormal = _mm512_testn_epi32_mask(e4m3_i32x16, _mm512_set1_epi32(0x78));
-    __m512 subnorm_lut_f32x16 = _mm512_setr_ps(                                                //
-        0, 1.0f / 512, 2.0f / 512, 3.0f / 512, 4.0f / 512, 5.0f / 512, 6.0f / 512, 7.0f / 512, //
-        0, 1.0f / 512, 2.0f / 512, 3.0f / 512, 4.0f / 512, 5.0f / 512, 6.0f / 512, 7.0f / 512);
-    __m512 subnorm_abs_f32x16 = _mm512_permutexvar_ps(mantissa_i32x16, subnorm_lut_f32x16);
-    result_f32x16 = _mm512_mask_or_ps(result_f32x16, is_subnormal, subnorm_abs_f32x16,
-                                      _mm512_castsi512_ps(sign_i32x16));
-
-    // NaN: E4M3FN has NaN only at magnitude 0x7F (single mask comparison)
-    __m512i lower7_i32x16 = _mm512_and_si512(e4m3_i32x16, _mm512_set1_epi32(0x7F));
-    __mmask16 is_nan = _mm512_cmpeq_epi32_mask(lower7_i32x16, _mm512_set1_epi32(0x7F));
-    __m512i nan_i32x16 = _mm512_or_si512(sign_i32x16, _mm512_set1_epi32(0x7FC00000));
-    return _mm512_mask_blend_ps(is_nan, result_f32x16, _mm512_castsi512_ps(nan_i32x16));
-}
-
-/** @brief Convert 16x e5m2 → 16x f32 via bit manipulation (AVX-512).
- *  E5M2 format: S EEEEE MM (bias=15). F32: sign<<31, (exp+112)<<23, mantissa<<21.
- *  Subnormals (exp=0): value = mantissa × 2⁽¹⁻¹⁵⁾ × 2⁻² = mantissa ÷ 65536. */
+    __m256i const magnitude_mask_u16x16 = _mm256_set1_epi16(0x7F);
+    __m256i const sign_mask_u16x16 = _mm256_set1_epi16((short)0x80);
+    __m256i const f16_nan_u16x16 = _mm256_set1_epi16(0x7E00);
+    __m256i word_u16x16 = _mm256_cvtepu8_epi16(e4m3_i8x16);
+    __m256i magnitude_u16x16 = _mm256_and_si256(word_u16x16, magnitude_mask_u16x16);
+    __mmask16 is_nan = _mm256_cmpeq_epi16_mask(magnitude_u16x16, magnitude_mask_u16x16);
+    __m256i shifted_magnitude_u16x16 = _mm256_slli_epi16(magnitude_u16x16, 7);
+    __m256i shifted_sign_u16x16 = _mm256_slli_epi16(_mm256_and_si256(word_u16x16, sign_mask_u16x16), 8);
+    __m256i f16_bits_u16x16 = _mm256_or_si256(shifted_magnitude_u16x16, shifted_sign_u16x16);
+    f16_bits_u16x16 = _mm256_mask_mov_epi16(f16_bits_u16x16, is_nan, f16_nan_u16x16);
+    __m512 fake_f32x16 = _mm512_cvtph_ps(f16_bits_u16x16);
+    return _mm512_mul_ps(fake_f32x16, _mm512_set1_ps(256.0f));
+}
+
+/** @brief Convert 16x e4m3 → 16x f16 via arithmetic + 8-entry subnormal LUT (AVX-512BW + AVX-512VL).
+ *  E4M3: S EEEE MMM (bias=7). F16: S EEEEE MMMMMMMMMM (bias=15).
+ *  Normal (exp != 0): F16 = ((lower7 << 7) + 0x2000) | (sign << 8) — bias delta 8 added at the
+ *  exp-position (8 << 10 = 0x2000) after placing magnitude bits at F16 positions 13..7.
+ *  Subnormal (exp == 0): looked up from 8-entry F16 LUT — values 0, 1/512, 2/512, …, 7/512 encoded as
+ *  F16 normals (the smallest E4M3 subnormal 1/512 = 2⁻⁹ is well within F16 normal range).
+ *  NaN (|byte| == 0x7F): blended in as F16 quiet NaN with original sign. */
+NK_INTERNAL __m256i nk_e4m3x16_to_f16x16_skylake_(__m128i e4m3_u8x16) {
+    __m256i e4m3_i16x16 = _mm256_cvtepu8_epi16(e4m3_u8x16);
+    __m256i sign_i16x16 = _mm256_and_si256(e4m3_i16x16, _mm256_set1_epi16((short)0x80));
+    __m256i lower7_i16x16 = _mm256_and_si256(e4m3_i16x16, _mm256_set1_epi16(0x7F));
+    __m256i normal_abs_i16x16 = _mm256_add_epi16(_mm256_slli_epi16(lower7_i16x16, 7), _mm256_set1_epi16(0x2000));
+    __m256i subn_lut_i16x16 = _mm256_set_epi16( //
+        0x2300, 0x2200, 0x2100, 0x2000, 0x1E00, 0x1C00, 0x1800, 0x0000, 0x2300, 0x2200, 0x2100, 0x2000, 0x1E00, 0x1C00,
+        0x1800, 0x0000);
+    __m256i mant_idx_i16x16 = _mm256_and_si256(e4m3_i16x16, _mm256_set1_epi16(0x07));
+    __m256i subn_abs_i16x16 = _mm256_permutexvar_epi16(mant_idx_i16x16, subn_lut_i16x16);
+    __mmask16 is_subnormal = _mm256_testn_epi16_mask(e4m3_i16x16, _mm256_set1_epi16(0x78));
+    __m256i abs_i16x16 = _mm256_mask_blend_epi16(is_subnormal, normal_abs_i16x16, subn_abs_i16x16);
+    __m256i shifted_sign_i16x16 = _mm256_slli_epi16(sign_i16x16, 8);
+    __m256i result_i16x16 = _mm256_or_si256(abs_i16x16, shifted_sign_i16x16);
+    __mmask16 is_nan = _mm256_cmpeq_epi16_mask(lower7_i16x16, _mm256_set1_epi16(0x7F));
+    __m256i nan_i16x16 = _mm256_or_si256(shifted_sign_i16x16, _mm256_set1_epi16(0x7E00));
+    return _mm256_mask_blend_epi16(is_nan, result_i16x16, nan_i16x16);
+}
+
+/** @brief Convert 16x e5m2 → 16x f32 via free-shift widen (AVX-512 + F16C).
+ *  E5M2 shares F16's exponent bias (15): `(byte << 8)` is the matching F16 bit
+ *  pattern for every E5M2 value (normals, subnormals, zero, ±Inf, NaN — all
+ *  bit-exact). Widen u8 → u16, shift, then VCVTPH2PS to F32. Three ops total. */
 NK_INTERNAL __m512 nk_e5m2x16_to_f32x16_skylake_(__m128i e5m2_i8x16) {
-    __m512i e5m2_i32x16 = _mm512_cvtepu8_epi32(e5m2_i8x16);
-
-    // Extract fields
-    __m512i exp_i32x16 = _mm512_and_si512(_mm512_srli_epi32(e5m2_i32x16, 2), _mm512_set1_epi32(0x1F));
-    __m512i mantissa_i32x16 = _mm512_and_si512(e5m2_i32x16, _mm512_set1_epi32(0x03));
-    __m512i sign_i32x16 = _mm512_slli_epi32(_mm512_srli_epi32(e5m2_i32x16, 7), 31);
-
-    // Normal path: sign | ((exp+112)<<23) | (mantissa<<21)
-    __m512i f32_exp_i32x16 = _mm512_slli_epi32(_mm512_add_epi32(exp_i32x16, _mm512_set1_epi32(112)), 23);
-    __m512i f32_mantissa_i32x16 = _mm512_slli_epi32(mantissa_i32x16, 21);
-    __m512 result_f32x16 = _mm512_castsi512_ps(
-        _mm512_ternarylogic_epi32(sign_i32x16, f32_exp_i32x16, f32_mantissa_i32x16, 0xFE));
-
-    // Subnormal fix: for exp==0 lanes, replace with (mantissa / 65536) | sign using masked OR
-    __mmask16 is_subnormal = _mm512_testn_epi32_mask(e5m2_i32x16, _mm512_set1_epi32(0x7C));
-    __m512 subnorm_abs_f32x16 = _mm512_mul_ps(_mm512_cvtepi32_ps(mantissa_i32x16), _mm512_set1_ps(1.0f / 65536.0f));
-    return _mm512_mask_or_ps(result_f32x16, is_subnormal, subnorm_abs_f32x16, _mm512_castsi512_ps(sign_i32x16));
+    __m256i e5m2_u16x16 = _mm256_cvtepu8_epi16(e5m2_i8x16);
+    __m256i f16_bits_u16x16 = _mm256_slli_epi16(e5m2_u16x16, 8);
+    return _mm512_cvtph_ps(f16_bits_u16x16);
 }
 
 /** @brief Convert 16x e2m3 → 16x f32 via bit manipulation (AVX-512).
@@ -660,6 +663,18 @@ NK_INTERNAL void nk_partial_load_e4m3x16_to_f32x16_skylake_(void const *src, nk_
     dst->zmm_ps = nk_e4m3x16_to_f32x16_skylake_(e4m3_partial.xmm);
 }
 
+/** @brief Load 16 e4m3 values and convert to 16 f16 (Skylake AVX-512BW). */
+NK_INTERNAL void nk_load_e4m3x16_to_f16x16_skylake_(void const *src, nk_b256_vec_t *dst) {
+    dst->ymm = nk_e4m3x16_to_f16x16_skylake_(_mm_loadu_si128((__m128i const *)src));
+}
+
+/** @brief Partial load of up to 16 e4m3 values with conversion to f16 (Skylake AVX-512BW). */
+NK_INTERNAL void nk_partial_load_e4m3x16_to_f16x16_skylake_(void const *src, nk_b256_vec_t *dst, nk_size_t n) {
+    nk_b128_vec_t e4m3_partial;
+    nk_partial_load_b8x16_skylake_(src, &e4m3_partial, n);
+    dst->ymm = nk_e4m3x16_to_f16x16_skylake_(e4m3_partial.xmm);
+}
+
 /** @brief Load 16 e5m2 values and convert to 16 f32 (Skylake AVX-512). */
 NK_INTERNAL void nk_load_e5m2x16_to_f32x16_skylake_(void const *src, nk_b512_vec_t *dst) {
     dst->zmm_ps = nk_e5m2x16_to_f32x16_skylake_(_mm_loadu_si128((__m128i const *)src));

package/include/numkong/cast.h

@@ -175,6 +175,7 @@ NK_PUBLIC void nk_cast_v128relaxed(void const *from, nk_dtype_t from_type, nk_si
 #include "numkong/cast/icelake.h"
 #include "numkong/cast/sapphire.h"
 #include "numkong/cast/rvv.h"
+#include "numkong/cast/v128relaxed.h"
 #include "numkong/cast/powervsx.h"
 #include "numkong/cast/loongsonasx.h"
 

package/include/numkong/curved/smef64.h

@@ -52,9 +52,10 @@
 #if NK_TARGET_SMEF64
 
 #include "numkong/types.h"
+#include "numkong/reduce/sve.h"    // `nk_svaddv_f64_`
 #include "numkong/spatial/neon.h"  // `nk_f64_sqrt_neon`
-#include "numkong/dots/sme.h"      // nk_sme_zero_za64_tile_0_, etc. (for f32 FMOPA)
-#include "numkong/curved/serial.h" // `nk_bilinear_f64_serial`, etc.
+#include "numkong/dots/sme.h"      // `nk_sme_zero_za64_tile_0_`
+#include "numkong/curved/serial.h" // `nk_bilinear_f64_serial`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -90,8 +91,8 @@ NK_PUBLIC void nk_dot2_f64_sve_accumulate_(svbool_t predicate_b64x, svfloat64_t
  *  @brief f32 bilinear: GEMV via FMOPA (widening f32→f64, exact accumulation).
  *  ZA0.D = C staging, ZA1.D = GEMV accumulator.
  */
-__arm_locally_streaming __arm_new("za") static void nk_bilinear_f32_smef64_streaming_(
-    nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t dimensions, nk_f64_t *result) {
+__arm_new("za") static void nk_bilinear_f32_smef64_streaming_( //
+    nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t dimensions, nk_f64_t *result) NK_STREAMING_ {
     svbool_t predicate_body_b64x = svptrue_b64();
     nk_size_t tile_dimension = svcntd();
     nk_f64_t outer_sum_f64 = 0.0;
@@ -124,24 +125,25 @@ __arm_locally_streaming __arm_new("za") static void nk_bilinear_f32_smef64_strea
         svfloat64_t v_f64x = svread_ver_za64_f64_m(svdup_f64(0.0), row_predicate_b64x, 1, 0);
         svfloat64_t a_f64x = svcvt_f64_f32_x(
             row_predicate_b64x, svreinterpret_f32_u64(svld1uw_u64(row_predicate_b64x, (nk_u32_t const *)(a + row))));
-        outer_sum_f64 += svaddv_f64(predicate_body_b64x, svmul_f64_x(row_predicate_b64x, a_f64x, v_f64x));
+        outer_sum_f64 += nk_svaddv_f64_(predicate_body_b64x, svmul_f64_x(row_predicate_b64x, a_f64x, v_f64x));
     }
 
     *result = outer_sum_f64;
 }
 
-NK_PUBLIC void nk_bilinear_f32_smef64(nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t dimensions,
-                                      nk_f64_t *result) {
+NK_PUBLIC void nk_bilinear_f32_smef64( //
+    nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t dimensions, nk_f64_t *result) {
+    nk_sme_start_streaming_();
     nk_bilinear_f32_smef64_streaming_(a, b, c, dimensions, result);
+    nk_sme_stop_streaming_();
 }
 
 /**
  *  @brief f32 Mahalanobis: GEMV v = C×d via FMOPA, where d = a − b (exact in f64).
  *  ZA0.D = C staging, ZA1.D = GEMV accumulator.
  */
-__arm_locally_streaming __arm_new("za") static nk_f64_t
-    nk_mahalanobis_f32_smef64_streaming_(nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c,
-                                         nk_size_t dimensions) {
+__arm_new("za") static nk_f64_t nk_mahalanobis_f32_smef64_streaming_( //
+    nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t dimensions) NK_STREAMING_ {
 
     svbool_t predicate_body_b64x = svptrue_b64();
     nk_size_t tile_dimension = svcntd();
@@ -179,15 +181,17 @@ __arm_locally_streaming __arm_new("za") static nk_f64_t
         svfloat64_t b_f64x = svcvt_f64_f32_x(
             row_predicate_b64x, svreinterpret_f32_u64(svld1uw_u64(row_predicate_b64x, (nk_u32_t const *)(b + row))));
         svfloat64_t d_f64x = svsub_f64_x(row_predicate_b64x, a_f64x, b_f64x);
-        outer_sum_f64 += svaddv_f64(predicate_body_b64x, svmul_f64_x(row_predicate_b64x, d_f64x, v_f64x));
+        outer_sum_f64 += nk_svaddv_f64_(predicate_body_b64x, svmul_f64_x(row_predicate_b64x, d_f64x, v_f64x));
     }
 
     return outer_sum_f64;
 }
 
-NK_PUBLIC void nk_mahalanobis_f32_smef64(nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t dimensions,
-                                         nk_f64_t *result) {
+NK_PUBLIC void nk_mahalanobis_f32_smef64( //
+    nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t dimensions, nk_f64_t *result) {
+    nk_sme_start_streaming_();
     nk_f64_t quadratic = nk_mahalanobis_f32_smef64_streaming_(a, b, c, dimensions);
+    nk_sme_stop_streaming_();
     *result = nk_f64_sqrt_neon(quadratic > 0 ? quadratic : 0);
 }
 
@@ -195,9 +199,8 @@ NK_PUBLIC void nk_mahalanobis_f32_smef64(nk_f32_t const *a, nk_f32_t const *b, n
  *  @brief f64 bilinear: row-by-row streaming SVE with Dot2 compensation.
  *  4-row fast path shares b_f64x loads; 1-row tail for remainder.
  */
-__arm_locally_streaming static void nk_bilinear_f64_smef64_streaming_(nk_f64_t const *a, nk_f64_t const *b,
-                                                                      nk_f64_t const *c, nk_size_t dimensions,
-                                                                      nk_f64_t *result) {
+static void nk_bilinear_f64_smef64_ssve_( //
+    nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t dimensions, nk_f64_t *result) NK_STREAMING_ {
     svbool_t predicate_all_b64x = svptrue_b64();
     nk_f64_t outer_sum = 0.0, outer_comp = 0.0;
     nk_size_t row = 0;
@@ -226,14 +229,18 @@ __arm_locally_streaming static void nk_bilinear_f64_smef64_streaming_(nk_f64_t c
             predicate_b64x = svwhilelt_b64(j, dimensions);
         }
 
-        nk_f64_dot2_(&outer_sum, &outer_comp, a0,
-                     svaddv_f64(predicate_all_b64x, sum_0_f64x) + svaddv_f64(predicate_all_b64x, compensation_0_f64x));
-        nk_f64_dot2_(&outer_sum, &outer_comp, a1,
-                     svaddv_f64(predicate_all_b64x, sum_1_f64x) + svaddv_f64(predicate_all_b64x, compensation_1_f64x));
-        nk_f64_dot2_(&outer_sum, &outer_comp, a2,
-                     svaddv_f64(predicate_all_b64x, sum_2_f64x) + svaddv_f64(predicate_all_b64x, compensation_2_f64x));
-        nk_f64_dot2_(&outer_sum, &outer_comp, a3,
-                     svaddv_f64(predicate_all_b64x, sum_3_f64x) + svaddv_f64(predicate_all_b64x, compensation_3_f64x));
+        nk_f64_dot2_(
+            &outer_sum, &outer_comp, a0,
+            nk_svaddv_f64_(predicate_all_b64x, sum_0_f64x) + nk_svaddv_f64_(predicate_all_b64x, compensation_0_f64x));
+        nk_f64_dot2_(
+            &outer_sum, &outer_comp, a1,
+            nk_svaddv_f64_(predicate_all_b64x, sum_1_f64x) + nk_svaddv_f64_(predicate_all_b64x, compensation_1_f64x));
+        nk_f64_dot2_(
+            &outer_sum, &outer_comp, a2,
+            nk_svaddv_f64_(predicate_all_b64x, sum_2_f64x) + nk_svaddv_f64_(predicate_all_b64x, compensation_2_f64x));
+        nk_f64_dot2_(
+            &outer_sum, &outer_comp, a3,
+            nk_svaddv_f64_(predicate_all_b64x, sum_3_f64x) + nk_svaddv_f64_(predicate_all_b64x, compensation_3_f64x));
     }
 
     // 1-row tail
@@ -250,24 +257,27 @@ __arm_locally_streaming static void nk_bilinear_f64_smef64_streaming_(nk_f64_t c
             predicate_b64x = svwhilelt_b64(j, dimensions);
         }
 
-        nk_f64_t cb_j = svaddv_f64(predicate_all_b64x, sum_f64x) + svaddv_f64(predicate_all_b64x, compensation_f64x);
+        nk_f64_t cb_j = nk_svaddv_f64_(predicate_all_b64x, sum_f64x) +
+                        nk_svaddv_f64_(predicate_all_b64x, compensation_f64x);
         nk_f64_dot2_(&outer_sum, &outer_comp, a[row], cb_j);
     }
 
     *result = outer_sum + outer_comp;
 }
 
-NK_PUBLIC void nk_bilinear_f64_smef64(nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t dimensions,
-                                      nk_f64_t *result) {
-    nk_bilinear_f64_smef64_streaming_(a, b, c, dimensions, result);
+NK_PUBLIC void nk_bilinear_f64_smef64( //
+    nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t dimensions, nk_f64_t *result) {
+    nk_sme_start_streaming_();
+    nk_bilinear_f64_smef64_ssve_(a, b, c, dimensions, result);
+    nk_sme_stop_streaming_();
 }
 
 /**
  *  @brief f64 Mahalanobis: row-by-row streaming SVE with Dot2 compensation.
  *  4-row fast path shares (a−b) column vector; 1-row tail for remainder.
  */
-__arm_locally_streaming static nk_f64_t nk_mahalanobis_f64_smef64_streaming_(nk_f64_t const *a, nk_f64_t const *b,
-                                                                             nk_f64_t const *c, nk_size_t dimensions) {
+static nk_f64_t nk_mahalanobis_f64_smef64_ssve_( //
+    nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t dimensions) NK_STREAMING_ {
     svbool_t predicate_all_b64x = svptrue_b64();
     nk_f64_t outer_sum = 0.0, outer_comp = 0.0;
     nk_size_t row = 0;
@@ -298,14 +308,18 @@ __arm_locally_streaming static nk_f64_t nk_mahalanobis_f64_smef64_streaming_(nk_
             predicate_b64x = svwhilelt_b64(j, dimensions);
         }
 
-        nk_f64_dot2_(&outer_sum, &outer_comp, d0,
-                     svaddv_f64(predicate_all_b64x, sum_0_f64x) + svaddv_f64(predicate_all_b64x, compensation_0_f64x));
-        nk_f64_dot2_(&outer_sum, &outer_comp, d1,
-                     svaddv_f64(predicate_all_b64x, sum_1_f64x) + svaddv_f64(predicate_all_b64x, compensation_1_f64x));
-        nk_f64_dot2_(&outer_sum, &outer_comp, d2,
-                     svaddv_f64(predicate_all_b64x, sum_2_f64x) + svaddv_f64(predicate_all_b64x, compensation_2_f64x));
-        nk_f64_dot2_(&outer_sum, &outer_comp, d3,
-                     svaddv_f64(predicate_all_b64x, sum_3_f64x) + svaddv_f64(predicate_all_b64x, compensation_3_f64x));
+        nk_f64_dot2_(
+            &outer_sum, &outer_comp, d0,
+            nk_svaddv_f64_(predicate_all_b64x, sum_0_f64x) + nk_svaddv_f64_(predicate_all_b64x, compensation_0_f64x));
+        nk_f64_dot2_(
+            &outer_sum, &outer_comp, d1,
+            nk_svaddv_f64_(predicate_all_b64x, sum_1_f64x) + nk_svaddv_f64_(predicate_all_b64x, compensation_1_f64x));
+        nk_f64_dot2_(
+            &outer_sum, &outer_comp, d2,
+            nk_svaddv_f64_(predicate_all_b64x, sum_2_f64x) + nk_svaddv_f64_(predicate_all_b64x, compensation_2_f64x));
+        nk_f64_dot2_(
+            &outer_sum, &outer_comp, d3,
+            nk_svaddv_f64_(predicate_all_b64x, sum_3_f64x) + nk_svaddv_f64_(predicate_all_b64x, compensation_3_f64x));
     }
 
     // 1-row tail
@@ -324,16 +338,19 @@ __arm_locally_streaming static nk_f64_t nk_mahalanobis_f64_smef64_streaming_(nk_
             predicate_b64x = svwhilelt_b64(j, dimensions);
         }
 
-        nk_f64_t cb_j = svaddv_f64(predicate_all_b64x, sum_f64x) + svaddv_f64(predicate_all_b64x, compensation_f64x);
+        nk_f64_t cb_j = nk_svaddv_f64_(predicate_all_b64x, sum_f64x) +
+                        nk_svaddv_f64_(predicate_all_b64x, compensation_f64x);
         nk_f64_dot2_(&outer_sum, &outer_comp, diff_row, cb_j);
     }
 
     return outer_sum + outer_comp;
 }
 
-NK_PUBLIC void nk_mahalanobis_f64_smef64(nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t dimensions,
-                                         nk_f64_t *result) {
-    nk_f64_t quadratic = nk_mahalanobis_f64_smef64_streaming_(a, b, c, dimensions);
+NK_PUBLIC void nk_mahalanobis_f64_smef64( //
+    nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t dimensions, nk_f64_t *result) {
+    nk_sme_start_streaming_();
+    nk_f64_t quadratic = nk_mahalanobis_f64_smef64_ssve_(a, b, c, dimensions);
+    nk_sme_stop_streaming_();
     *result = nk_f64_sqrt_neon(quadratic > 0 ? quadratic : 0);
 }
 
@@ -341,11 +358,9 @@ NK_PUBLIC void nk_mahalanobis_f64_smef64(nk_f64_t const *a, nk_f64_t const *b, n
  *  @brief f32c bilinear: complex GEMV via FMOPA (widening f32→f64).
  *  ZA0.D = C staging, ZA1.D = v_real accumulator, ZA2.D = v_imag accumulator.
  */
-__arm_locally_streaming __arm_new("za") static void nk_bilinear_f32c_smef64_streaming_(nk_f32c_t const *a_pairs,
-                                                                                       nk_f32c_t const *b_pairs,
-                                                                                       nk_f32c_t const *c_pairs,
-                                                                                       nk_size_t dimensions,
-                                                                                       nk_f64c_t *results) {
+__arm_new("za") static void nk_bilinear_f32c_smef64_streaming_( //
+    nk_f32c_t const *a_pairs, nk_f32c_t const *b_pairs, nk_f32c_t const *c_pairs, nk_size_t dimensions,
+    nk_f64c_t *results) NK_STREAMING_ {
     svbool_t predicate_body_b64x = svptrue_b64();
     nk_size_t tile_dimension = svcntd();
     nk_f64_t outer_sum_real_f64 = 0.0, outer_sum_imag_f64 = 0.0;
@@ -407,10 +422,10 @@ __arm_locally_streaming __arm_new("za") static void nk_bilinear_f32c_smef64_stre
         svfloat64_t a_im_f64x = svcvt_f64_f32_x(row_predicate_b64x, svtrn2_f32(a_f32x, a_f32x));
 
         // Complex dot: a × v
-        outer_sum_real_f64 += svaddv_f64(
+        outer_sum_real_f64 += nk_svaddv_f64_(
             predicate_body_b64x, svsub_f64_x(row_predicate_b64x, svmul_f64_x(row_predicate_b64x, a_re_f64x, v_re_f64x),
                                              svmul_f64_x(row_predicate_b64x, a_im_f64x, v_im_f64x)));
-        outer_sum_imag_f64 += svaddv_f64(
+        outer_sum_imag_f64 += nk_svaddv_f64_(
             predicate_body_b64x, svadd_f64_x(row_predicate_b64x, svmul_f64_x(row_predicate_b64x, a_re_f64x, v_im_f64x),
                                              svmul_f64_x(row_predicate_b64x, a_im_f64x, v_re_f64x)));
     }
@@ -419,19 +434,21 @@ __arm_locally_streaming __arm_new("za") static void nk_bilinear_f32c_smef64_stre
     results->imag = outer_sum_imag_f64;
 }
 
-NK_PUBLIC void nk_bilinear_f32c_smef64(nk_f32c_t const *a_pairs, nk_f32c_t const *b_pairs, nk_f32c_t const *c_pairs,
-                                       nk_size_t dimensions, nk_f64c_t *results) {
+NK_PUBLIC void nk_bilinear_f32c_smef64( //
+    nk_f32c_t const *a_pairs, nk_f32c_t const *b_pairs, nk_f32c_t const *c_pairs, nk_size_t dimensions,
+    nk_f64c_t *results) {
+    nk_sme_start_streaming_();
     nk_bilinear_f32c_smef64_streaming_(a_pairs, b_pairs, c_pairs, dimensions, results);
+    nk_sme_stop_streaming_();
 }
 
 /**
  *  @brief f64c bilinear: interleaved Dot2 with permute + deferred XOR sign-flip.
  *  2 accumulators instead of 4, halving inner loop work (~15 vs ~28 SVE ops).
  */
-__arm_locally_streaming static void nk_bilinear_f64c_smef64_streaming_(nk_f64c_t const *a_pairs,
-                                                                       nk_f64c_t const *b_pairs,
-                                                                       nk_f64c_t const *c_pairs, nk_size_t dimensions,
-                                                                       nk_f64c_t *results) {
+static void nk_bilinear_f64c_smef64_ssve_( //
+    nk_f64c_t const *a_pairs, nk_f64c_t const *b_pairs, nk_f64c_t const *c_pairs, nk_size_t dimensions,
+    nk_f64c_t *results) NK_STREAMING_ {
     svbool_t predicate_all_b64x = svptrue_b64();
     nk_f64_t outer_sum_real = 0.0, outer_comp_real = 0.0;
     nk_f64_t outer_sum_imag = 0.0, outer_comp_imag = 0.0;
@@ -474,10 +491,10 @@ __arm_locally_streaming static void nk_bilinear_f64c_smef64_streaming_(nk_f64c_t
             sveor_u64_x(predicate_all_b64x, svreinterpret_u64_f64(sum_real_f64x), sign_mask_u64x));
         comp_real_f64x = svreinterpret_f64_u64(
             sveor_u64_x(predicate_all_b64x, svreinterpret_u64_f64(comp_real_f64x), sign_mask_u64x));
-        nk_f64_t inner_real = svaddv_f64(predicate_all_b64x,
-                                         svadd_f64_x(predicate_all_b64x, sum_real_f64x, comp_real_f64x));
-        nk_f64_t inner_imag = svaddv_f64(predicate_all_b64x,
-                                         svadd_f64_x(predicate_all_b64x, sum_imag_f64x, comp_imag_f64x));
+        nk_f64_t inner_real = nk_svaddv_f64_(predicate_all_b64x,
+                                             svadd_f64_x(predicate_all_b64x, sum_real_f64x, comp_real_f64x));
+        nk_f64_t inner_imag = nk_svaddv_f64_(predicate_all_b64x,
+                                             svadd_f64_x(predicate_all_b64x, sum_imag_f64x, comp_imag_f64x));
 
         // Outer Dot2 complex multiply: a × inner
         nk_f64_dot2_(&outer_sum_real, &outer_comp_real, a_real, inner_real);
@@ -490,9 +507,12 @@ __arm_locally_streaming static void nk_bilinear_f64c_smef64_streaming_(nk_f64c_t
     results->imag = outer_sum_imag + outer_comp_imag;
 }
 
-NK_PUBLIC void nk_bilinear_f64c_smef64(nk_f64c_t const *a_pairs, nk_f64c_t const *b_pairs, nk_f64c_t const *c_pairs,
-                                       nk_size_t dimensions, nk_f64c_t *results) {
-    nk_bilinear_f64c_smef64_streaming_(a_pairs, b_pairs, c_pairs, dimensions, results);
+NK_PUBLIC void nk_bilinear_f64c_smef64( //
+    nk_f64c_t const *a_pairs, nk_f64c_t const *b_pairs, nk_f64c_t const *c_pairs, nk_size_t dimensions,
+    nk_f64c_t *results) {
+    nk_sme_start_streaming_();
+    nk_bilinear_f64c_smef64_ssve_(a_pairs, b_pairs, c_pairs, dimensions, results);
+    nk_sme_stop_streaming_();
 }
 
 #if defined(__clang__)

package/include/numkong/dot/README.md

@@ -111,6 +111,7 @@ This processes 64 E4M3 bytes per iteration in u8, doubling the element density o
 
 `nk_dot_e5m2_genoa` converts FP8 values to BF16, then accumulates via `VDPBF16PS`, reusing Genoa's BF16 dot-product instruction for FP8 types.
 Each `VDPBF16PS` fuses two BF16 multiply-adds per 32-bit lane at 6-cycle throughput.
+On Skylake-X–class CPUs without BF16 dot-product hardware, `nk_dot_e4m3_skylake` / `nk_dot_e5m2_skylake` (and their Haswell twins `nk_dot_e4m3_haswell` / `nk_dot_e5m2_haswell`) instead route through the Giesen-style FP8 → F16 fake-bit-pattern cast, widen via `VCVTPH2PS`, and accumulate in F32 with two independent FMA chains reducing into a single register — avoiding the 3-chain scheduler-stall of the BF16 algebraic form on kernels without native BF16 FMA.
 `nk_dot_bf16c_genoa` uses the same instruction for complex BF16, preparing operands with `VPSHUFB` for lane swapping and `VPXORD` with `0x80000000` for sign flips before feeding into `VDPBF16PS`.
 
 ### Deferred Sign-Flip in Complex Dot Products

package/include/numkong/dot/haswell.h

@@ -698,25 +698,39 @@ nk_dot_e4m3_haswell_cycle:
 
 NK_PUBLIC void nk_dot_e5m2_haswell(nk_e5m2_t const *a_scalars, nk_e5m2_t const *b_scalars, nk_size_t count_scalars,
                                    nk_f32_t *result) {
-    __m256 a_f32x8, b_f32x8;
-    __m256 sum_f32x8 = _mm256_setzero_ps();
+    // E5M2 shares F16 bias; inline the free-shift unpack for the two 8-lane halves.
+    __m256 first_chain_f32x8 = _mm256_setzero_ps();
+    __m256 second_chain_f32x8 = _mm256_setzero_ps();
+    __m128i const zero_u8x16 = _mm_setzero_si128();
+    __m128i a_u8x16, b_u8x16;
+
 nk_dot_e5m2_haswell_cycle:
-    if (count_scalars < 8) {
-        nk_b256_vec_t a_vec, b_vec;
-        nk_partial_load_e5m2x8_to_f32x8_haswell_(a_scalars, &a_vec, count_scalars);
-        nk_partial_load_e5m2x8_to_f32x8_haswell_(b_scalars, &b_vec, count_scalars);
-        a_f32x8 = a_vec.ymm_ps;
-        b_f32x8 = b_vec.ymm_ps;
+    if (count_scalars < 16) {
+        nk_b128_vec_t a_vec, b_vec;
+        nk_partial_load_b8x16_serial_(a_scalars, &a_vec, count_scalars);
+        nk_partial_load_b8x16_serial_(b_scalars, &b_vec, count_scalars);
+        a_u8x16 = a_vec.xmm;
+        b_u8x16 = b_vec.xmm;
         count_scalars = 0;
     }
     else {
-        a_f32x8 = nk_e5m2x8_to_f32x8_haswell_(_mm_loadl_epi64((__m128i const *)a_scalars));
-        b_f32x8 = nk_e5m2x8_to_f32x8_haswell_(_mm_loadl_epi64((__m128i const *)b_scalars));
-        a_scalars += 8, b_scalars += 8, count_scalars -= 8;
+        a_u8x16 = _mm_loadu_si128((__m128i const *)a_scalars);
+        b_u8x16 = _mm_loadu_si128((__m128i const *)b_scalars);
+        a_scalars += 16, b_scalars += 16, count_scalars -= 16;
     }
-    sum_f32x8 = _mm256_fmadd_ps(a_f32x8, b_f32x8, sum_f32x8);
+    __m128i a_even_f16x8 = _mm_unpacklo_epi8(zero_u8x16, a_u8x16);
+    __m128i a_odd_f16x8 = _mm_unpackhi_epi8(zero_u8x16, a_u8x16);
+    __m128i b_even_f16x8 = _mm_unpacklo_epi8(zero_u8x16, b_u8x16);
+    __m128i b_odd_f16x8 = _mm_unpackhi_epi8(zero_u8x16, b_u8x16);
+    __m256 a_first_f32x8 = _mm256_cvtph_ps(a_even_f16x8);
+    __m256 a_second_f32x8 = _mm256_cvtph_ps(a_odd_f16x8);
+    __m256 b_first_f32x8 = _mm256_cvtph_ps(b_even_f16x8);
+    __m256 b_second_f32x8 = _mm256_cvtph_ps(b_odd_f16x8);
+    first_chain_f32x8 = _mm256_fmadd_ps(a_first_f32x8, b_first_f32x8, first_chain_f32x8);
+    second_chain_f32x8 = _mm256_fmadd_ps(a_second_f32x8, b_second_f32x8, second_chain_f32x8);
     if (count_scalars) goto nk_dot_e5m2_haswell_cycle;
-    *result = (nk_f32_t)nk_reduce_add_f32x8_haswell_(sum_f32x8);
+
+    *result = (nk_f32_t)nk_reduce_add_f32x8_haswell_(_mm256_add_ps(first_chain_f32x8, second_chain_f32x8));
 }
 
 NK_PUBLIC void nk_dot_e2m3_haswell(nk_e2m3_t const *a_scalars, nk_e2m3_t const *b_scalars, nk_size_t count_scalars,
@@ -910,6 +924,71 @@ NK_INTERNAL void nk_dot_through_f32_update_haswell_(nk_dot_through_f32_state_has
     state->sum_f32x8 = _mm256_fmadd_ps(a.ymm_ps, b.ymm_ps, state->sum_f32x8);
 }
 
+/**
+ *  @brief E5M2 byte-batched update: consumes 32 raw E5M2 bytes per call and widens inline.
+ *  Two independent FMA chains (each 2-deep) merge into the single __m256 state accumulator.
+ */
+NK_INTERNAL void nk_dot_e5m2x32_update_haswell_(nk_dot_through_f32_state_haswell_t_ *state, nk_b256_vec_t a_bytes,
+                                                nk_b256_vec_t b_bytes, nk_size_t depth_offset,
+                                                nk_size_t active_dimensions) {
+    nk_unused_(depth_offset);
+    nk_unused_(active_dimensions);
+    __m128i const zero_u8x16 = _mm_setzero_si128();
+    __m128i a_low_u8x16 = _mm256_castsi256_si128(a_bytes.ymm);
+    __m128i a_high_u8x16 = _mm256_extracti128_si256(a_bytes.ymm, 1);
+    __m128i b_low_u8x16 = _mm256_castsi256_si128(b_bytes.ymm);
+    __m128i b_high_u8x16 = _mm256_extracti128_si256(b_bytes.ymm, 1);
+    __m128i a_first_f16x8 = _mm_unpacklo_epi8(zero_u8x16, a_low_u8x16);
+    __m128i a_second_f16x8 = _mm_unpackhi_epi8(zero_u8x16, a_low_u8x16);
+    __m128i a_third_f16x8 = _mm_unpacklo_epi8(zero_u8x16, a_high_u8x16);
+    __m128i a_fourth_f16x8 = _mm_unpackhi_epi8(zero_u8x16, a_high_u8x16);
+    __m128i b_first_f16x8 = _mm_unpacklo_epi8(zero_u8x16, b_low_u8x16);
+    __m128i b_second_f16x8 = _mm_unpackhi_epi8(zero_u8x16, b_low_u8x16);
+    __m128i b_third_f16x8 = _mm_unpacklo_epi8(zero_u8x16, b_high_u8x16);
+    __m128i b_fourth_f16x8 = _mm_unpackhi_epi8(zero_u8x16, b_high_u8x16);
+    __m256 a_first_f32x8 = _mm256_cvtph_ps(a_first_f16x8);
+    __m256 a_second_f32x8 = _mm256_cvtph_ps(a_second_f16x8);
+    __m256 a_third_f32x8 = _mm256_cvtph_ps(a_third_f16x8);
+    __m256 a_fourth_f32x8 = _mm256_cvtph_ps(a_fourth_f16x8);
+    __m256 b_first_f32x8 = _mm256_cvtph_ps(b_first_f16x8);
+    __m256 b_second_f32x8 = _mm256_cvtph_ps(b_second_f16x8);
+    __m256 b_third_f32x8 = _mm256_cvtph_ps(b_third_f16x8);
+    __m256 b_fourth_f32x8 = _mm256_cvtph_ps(b_fourth_f16x8);
+    __m256 first_chain_f32x8 = _mm256_mul_ps(a_first_f32x8, b_first_f32x8);
+    __m256 second_chain_f32x8 = _mm256_mul_ps(a_second_f32x8, b_second_f32x8);
+    first_chain_f32x8 = _mm256_fmadd_ps(a_third_f32x8, b_third_f32x8, first_chain_f32x8);
+    second_chain_f32x8 = _mm256_fmadd_ps(a_fourth_f32x8, b_fourth_f32x8, second_chain_f32x8);
+    state->sum_f32x8 = _mm256_add_ps(state->sum_f32x8, _mm256_add_ps(first_chain_f32x8, second_chain_f32x8));
+}
+
+/**
+ *  @brief E4M3 byte-batched update: consumes 32 raw E4M3 bytes per call. Widens through the
+ *  Giesen cast helper. Two independent FMA chains merge into the single state accumulator.
+ */
+NK_INTERNAL void nk_dot_e4m3x32_update_haswell_(nk_dot_through_f32_state_haswell_t_ *state, nk_b256_vec_t a_bytes,
+                                                nk_b256_vec_t b_bytes, nk_size_t depth_offset,
+                                                nk_size_t active_dimensions) {
+    nk_unused_(depth_offset);
+    nk_unused_(active_dimensions);
+    __m128i a_low_u8x16 = _mm256_castsi256_si128(a_bytes.ymm);
+    __m128i a_high_u8x16 = _mm256_extracti128_si256(a_bytes.ymm, 1);
+    __m128i b_low_u8x16 = _mm256_castsi256_si128(b_bytes.ymm);
+    __m128i b_high_u8x16 = _mm256_extracti128_si256(b_bytes.ymm, 1);
+    __m256 a_first_f32x8 = nk_e4m3x8_to_f32x8_haswell_(a_low_u8x16);
+    __m256 a_second_f32x8 = nk_e4m3x8_to_f32x8_haswell_(_mm_unpackhi_epi64(a_low_u8x16, a_low_u8x16));
+    __m256 a_third_f32x8 = nk_e4m3x8_to_f32x8_haswell_(a_high_u8x16);
+    __m256 a_fourth_f32x8 = nk_e4m3x8_to_f32x8_haswell_(_mm_unpackhi_epi64(a_high_u8x16, a_high_u8x16));
+    __m256 b_first_f32x8 = nk_e4m3x8_to_f32x8_haswell_(b_low_u8x16);
+    __m256 b_second_f32x8 = nk_e4m3x8_to_f32x8_haswell_(_mm_unpackhi_epi64(b_low_u8x16, b_low_u8x16));
+    __m256 b_third_f32x8 = nk_e4m3x8_to_f32x8_haswell_(b_high_u8x16);
+    __m256 b_fourth_f32x8 = nk_e4m3x8_to_f32x8_haswell_(_mm_unpackhi_epi64(b_high_u8x16, b_high_u8x16));
+    __m256 first_chain_f32x8 = _mm256_mul_ps(a_first_f32x8, b_first_f32x8);
+    __m256 second_chain_f32x8 = _mm256_mul_ps(a_second_f32x8, b_second_f32x8);
+    first_chain_f32x8 = _mm256_fmadd_ps(a_third_f32x8, b_third_f32x8, first_chain_f32x8);
+    second_chain_f32x8 = _mm256_fmadd_ps(a_fourth_f32x8, b_fourth_f32x8, second_chain_f32x8);
+    state->sum_f32x8 = _mm256_add_ps(state->sum_f32x8, _mm256_add_ps(first_chain_f32x8, second_chain_f32x8));
+}
+
 /**
  *  @brief Finalizes 4x low-precision dot-products placing them into 4x consecutive 32-bit slots.
  *  @sa nk_dot_f16x8_finalize_haswell, nk_dot_bf16x8_finalize_haswell

package/include/numkong/dot/rvvbf16.h

@@ -22,7 +22,7 @@
 #if NK_TARGET_RVVBF16
 
 #include "numkong/types.h"
-#include "numkong/cast/rvv.h" // `nk_e4m3m1_to_bf16m2_rvv_`, `nk_e5m2m1_to_bf16m2_rvv_`, etc.
+#include "numkong/cast/rvv.h" // `nk_e4m3m1_to_bf16m2_rvv_`, `nk_e5m2m1_to_bf16m2_rvv_`
 
 #if defined(__clang__)
 #pragma clang attribute push(__attribute__((target("arch=+v,+zvfbfwma"))), apply_to = function)

package/include/numkong/dot/rvvhalf.h

@@ -23,7 +23,7 @@
 #if NK_TARGET_RVVHALF
 
 #include "numkong/types.h"
-#include "numkong/cast/rvv.h" // `nk_e4m3m1_to_f16m2_rvv_`, `nk_e2m3m1_to_f16m2_rvv_`, etc.
+#include "numkong/cast/rvv.h" // `nk_e4m3m1_to_f16m2_rvv_`, `nk_e2m3m1_to_f16m2_rvv_`
 
 #if defined(__clang__)
 #pragma clang attribute push(__attribute__((target("arch=+v,+zvfh"))), apply_to = function)

package/include/numkong/dot/serial.h

@@ -139,6 +139,15 @@ extern "C" {
         result->imag = sum_imag;                                                                            \
     }
 
+/*  Keep the serial instantiations below actually scalar, regardless of build type.
+ *  See dots/serial.h for rationale. */
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((noinline)), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC optimize("no-tree-vectorize", "no-tree-slp-vectorize", "no-ipa-cp-clone", "no-inline")
+#endif
+
 #pragma region F32 and F64 Floats
 
 nk_define_dot_(f32, f64, f64, nk_assign_from_to_)            // nk_dot_f32_serial
@@ -867,6 +876,12 @@ NK_INTERNAL nk_i32_t nk_sum_i4x32_finalize_serial(nk_sum_i4x32_state_serial_t co
 
 #pragma endregion Stateful Element Sum Helpers
 
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
 #if defined(__cplusplus)
 } // extern "C"
 #endif