numkong 7.4.5 → 7.6.0

package/include/numkong/dot/skylake.h

@@ -156,6 +156,36 @@ NK_INTERNAL void nk_dot_through_f32_update_skylake_(nk_dot_through_f32_state_sky
     state->sum_f32x16 = _mm512_fmadd_ps(a.zmm_ps, b.zmm_ps, state->sum_f32x16);
 }
 
+/**
+ *  @brief E5M2 byte-batched update: consumes 64 raw E5M2 bytes per call and widens inline.
+ *  Two independent FMA chains (each 2-deep) merge into the single state accumulator at exit.
+ *  Keeps register pressure at one __m512 across calls while breaking the FMA dep chain.
+ */
+NK_INTERNAL void nk_dot_e5m2x64_update_skylake_(nk_dot_through_f32_state_skylake_t_ *state, nk_b512_vec_t a_bytes,
+                                                nk_b512_vec_t b_bytes, nk_size_t depth_offset,
+                                                nk_size_t active_dimensions) {
+    nk_unused_(depth_offset);
+    nk_unused_(active_dimensions);
+    __m512i const zero_u8x64 = _mm512_setzero_si512();
+    __m512i a_even_f16x32 = _mm512_unpacklo_epi8(zero_u8x64, a_bytes.zmm);
+    __m512i a_odd_f16x32 = _mm512_unpackhi_epi8(zero_u8x64, a_bytes.zmm);
+    __m512i b_even_f16x32 = _mm512_unpacklo_epi8(zero_u8x64, b_bytes.zmm);
+    __m512i b_odd_f16x32 = _mm512_unpackhi_epi8(zero_u8x64, b_bytes.zmm);
+    __m512 a_first_f32x16 = _mm512_cvtph_ps(_mm512_castsi512_si256(a_even_f16x32));
+    __m512 a_second_f32x16 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(a_even_f16x32, 1));
+    __m512 a_third_f32x16 = _mm512_cvtph_ps(_mm512_castsi512_si256(a_odd_f16x32));
+    __m512 a_fourth_f32x16 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(a_odd_f16x32, 1));
+    __m512 b_first_f32x16 = _mm512_cvtph_ps(_mm512_castsi512_si256(b_even_f16x32));
+    __m512 b_second_f32x16 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(b_even_f16x32, 1));
+    __m512 b_third_f32x16 = _mm512_cvtph_ps(_mm512_castsi512_si256(b_odd_f16x32));
+    __m512 b_fourth_f32x16 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(b_odd_f16x32, 1));
+    __m512 first_chain_f32x16 = _mm512_mul_ps(a_first_f32x16, b_first_f32x16);
+    __m512 second_chain_f32x16 = _mm512_mul_ps(a_second_f32x16, b_second_f32x16);
+    first_chain_f32x16 = _mm512_fmadd_ps(a_third_f32x16, b_third_f32x16, first_chain_f32x16);
+    second_chain_f32x16 = _mm512_fmadd_ps(a_fourth_f32x16, b_fourth_f32x16, second_chain_f32x16);
+    state->sum_f32x16 = _mm512_add_ps(state->sum_f32x16, _mm512_add_ps(first_chain_f32x16, second_chain_f32x16));
+}
+
 /**
  *  @brief Finalizes 4x low-precision dot-products placing them into 4x consecutive 32-bit slots.
  *  @sa nk_dot_f16x16_udpate_skylake, nk_dot_bf16x16_udpate_skylake
@@ -543,7 +573,7 @@ NK_PUBLIC void nk_dot_e4m3_skylake(nk_e4m3_t const *a_scalars, nk_e4m3_t const *
 
 nk_dot_e4m3_skylake_cycle:
     if (count_scalars < 16) {
-        __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, count_scalars);
+        __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, (unsigned int)count_scalars);
         a_e4m3_u8x16 = _mm_maskz_loadu_epi8(mask, a_scalars);
         b_e4m3_u8x16 = _mm_maskz_loadu_epi8(mask, b_scalars);
         count_scalars = 0;
@@ -563,27 +593,44 @@ nk_dot_e4m3_skylake_cycle:
 
 NK_PUBLIC void nk_dot_e5m2_skylake(nk_e5m2_t const *a_scalars, nk_e5m2_t const *b_scalars, nk_size_t count_scalars,
                                    nk_f32_t *result) {
-    __m128i a_e5m2_u8x16, b_e5m2_u8x16;
-    __m512 sum_f32x16 = _mm512_setzero_ps();
+    // E5M2 shares F16 bias (15): vpunpck*bw against zero places the byte as F16 encoding,
+    // so we inline the widen rather than calling the helper 4× — same ops, cleaner code.
+    __m512 first_chain_f32x16 = _mm512_setzero_ps();
+    __m512 second_chain_f32x16 = _mm512_setzero_ps();
+    __m512i const zero_u8x64 = _mm512_setzero_si512();
+    __m512i a_u8x64, b_u8x64;
 
 nk_dot_e5m2_skylake_cycle:
-    if (count_scalars < 16) {
-        __mmask16 mask = (__mmask16)_bzhi_u32(0xFFFF, count_scalars);
-        a_e5m2_u8x16 = _mm_maskz_loadu_epi8(mask, a_scalars);
-        b_e5m2_u8x16 = _mm_maskz_loadu_epi8(mask, b_scalars);
+    if (count_scalars < 64) {
+        __mmask64 mask = _bzhi_u64(0xFFFFFFFFFFFFFFFFULL, (unsigned int)count_scalars);
+        a_u8x64 = _mm512_maskz_loadu_epi8(mask, a_scalars);
+        b_u8x64 = _mm512_maskz_loadu_epi8(mask, b_scalars);
         count_scalars = 0;
     }
     else {
-        a_e5m2_u8x16 = _mm_loadu_si128((__m128i const *)a_scalars);
-        b_e5m2_u8x16 = _mm_loadu_si128((__m128i const *)b_scalars);
-        a_scalars += 16, b_scalars += 16, count_scalars -= 16;
+        a_u8x64 = _mm512_loadu_si512((__m512i const *)a_scalars);
+        b_u8x64 = _mm512_loadu_si512((__m512i const *)b_scalars);
+        a_scalars += 64, b_scalars += 64, count_scalars -= 64;
     }
-    __m512 a_f32x16 = nk_e5m2x16_to_f32x16_skylake_(a_e5m2_u8x16);
-    __m512 b_f32x16 = nk_e5m2x16_to_f32x16_skylake_(b_e5m2_u8x16);
-    sum_f32x16 = _mm512_fmadd_ps(a_f32x16, b_f32x16, sum_f32x16);
+    __m512i a_even_f16x32 = _mm512_unpacklo_epi8(zero_u8x64, a_u8x64);
+    __m512i a_odd_f16x32 = _mm512_unpackhi_epi8(zero_u8x64, a_u8x64);
+    __m512i b_even_f16x32 = _mm512_unpacklo_epi8(zero_u8x64, b_u8x64);
+    __m512i b_odd_f16x32 = _mm512_unpackhi_epi8(zero_u8x64, b_u8x64);
+    __m512 a_first_f32x16 = _mm512_cvtph_ps(_mm512_castsi512_si256(a_even_f16x32));
+    __m512 a_second_f32x16 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(a_even_f16x32, 1));
+    __m512 a_third_f32x16 = _mm512_cvtph_ps(_mm512_castsi512_si256(a_odd_f16x32));
+    __m512 a_fourth_f32x16 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(a_odd_f16x32, 1));
+    __m512 b_first_f32x16 = _mm512_cvtph_ps(_mm512_castsi512_si256(b_even_f16x32));
+    __m512 b_second_f32x16 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(b_even_f16x32, 1));
+    __m512 b_third_f32x16 = _mm512_cvtph_ps(_mm512_castsi512_si256(b_odd_f16x32));
+    __m512 b_fourth_f32x16 = _mm512_cvtph_ps(_mm512_extracti64x4_epi64(b_odd_f16x32, 1));
+    first_chain_f32x16 = _mm512_fmadd_ps(a_first_f32x16, b_first_f32x16, first_chain_f32x16);
+    second_chain_f32x16 = _mm512_fmadd_ps(a_second_f32x16, b_second_f32x16, second_chain_f32x16);
+    first_chain_f32x16 = _mm512_fmadd_ps(a_third_f32x16, b_third_f32x16, first_chain_f32x16);
+    second_chain_f32x16 = _mm512_fmadd_ps(a_fourth_f32x16, b_fourth_f32x16, second_chain_f32x16);
     if (count_scalars) goto nk_dot_e5m2_skylake_cycle;
 
-    *result = nk_reduce_add_f32x16_skylake_(sum_f32x16);
+    *result = nk_reduce_add_f32x16_skylake_(_mm512_add_ps(first_chain_f32x16, second_chain_f32x16));
 }
 
 NK_PUBLIC void nk_dot_e2m3_skylake(nk_e2m3_t const *a_scalars, nk_e2m3_t const *b_scalars, nk_size_t count_scalars,

package/include/numkong/dot/sve.h

@@ -39,6 +39,7 @@
 #if NK_TARGET_SVE
 
 #include "numkong/types.h"      // `nk_f32_t`
+#include "numkong/reduce/sve.h" // `nk_svaddv_f64_`
 #include "numkong/dot/serial.h" // `nk_u1x8_popcount_`
 
 #if defined(__cplusplus)
@@ -110,7 +111,7 @@ NK_PUBLIC void nk_dot_f32_sve(nk_f32_t const *a_scalars, nk_f32_t const *b_scala
         ab_f64x = svmla_f64_m(pred_odd_b64x, ab_f64x, svcvt_f64_f32_x(pred_odd_b64x, svext_f32(a_f32x, a_f32x, 1)),
                               svcvt_f64_f32_x(pred_odd_b64x, svext_f32(b_f32x, b_f32x, 1)));
     }
-    *result = svaddv_f64(svptrue_b64(), ab_f64x);
+    *result = nk_svaddv_f64_(svptrue_b64(), ab_f64x);
 }
 
 NK_PUBLIC void nk_dot_f32c_sve(nk_f32c_t const *a_pairs, nk_f32c_t const *b_pairs, nk_size_t count_pairs,
@@ -149,8 +150,8 @@ NK_PUBLIC void nk_dot_f32c_sve(nk_f32c_t const *a_pairs, nk_f32c_t const *b_pair
         ab_imag_f64x = svmla_f64_m(pred_odd_b64x, ab_imag_f64x, a_real_odd_f64x, b_imag_odd_f64x);
         ab_imag_f64x = svmla_f64_m(pred_odd_b64x, ab_imag_f64x, a_imag_odd_f64x, b_real_odd_f64x);
     }
-    results->real = svaddv_f64(svptrue_b64(), ab_real_f64x);
-    results->imag = svaddv_f64(svptrue_b64(), ab_imag_f64x);
+    results->real = nk_svaddv_f64_(svptrue_b64(), ab_real_f64x);
+    results->imag = nk_svaddv_f64_(svptrue_b64(), ab_imag_f64x);
 }
 
 NK_PUBLIC void nk_vdot_f32c_sve(nk_f32c_t const *a_pairs, nk_f32c_t const *b_pairs, nk_size_t count_pairs,
@@ -189,8 +190,8 @@ NK_PUBLIC void nk_vdot_f32c_sve(nk_f32c_t const *a_pairs, nk_f32c_t const *b_pai
         ab_imag_f64x = svmla_f64_m(pred_odd_b64x, ab_imag_f64x, a_real_odd_f64x, b_imag_odd_f64x);
         ab_imag_f64x = svmls_f64_m(pred_odd_b64x, ab_imag_f64x, a_imag_odd_f64x, b_real_odd_f64x);
     }
-    results->real = svaddv_f64(svptrue_b64(), ab_real_f64x);
-    results->imag = svaddv_f64(svptrue_b64(), ab_imag_f64x);
+    results->real = nk_svaddv_f64_(svptrue_b64(), ab_real_f64x);
+    results->imag = nk_svaddv_f64_(svptrue_b64(), ab_imag_f64x);
 }
 
 NK_PUBLIC void nk_dot_f64_sve(nk_f64_t const *a_scalars, nk_f64_t const *b_scalars, nk_size_t count_scalars,

package/include/numkong/dot/svebfdot.h

@@ -31,6 +31,7 @@
 #if NK_TARGET_SVEBFDOT
 
 #include "numkong/types.h"
+#include "numkong/reduce/sve.h" // `nk_svaddv_f64_`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -56,7 +57,7 @@ NK_PUBLIC void nk_dot_bf16_svebfdot(nk_bf16_t const *a_scalars, nk_bf16_t const
         sum_f32x = svbfdot_f32(sum_f32x, a_bf16x, b_bf16x);
         idx_scalars += svcnth();
     } while (idx_scalars < count_scalars);
-    *result = svaddv_f32(svptrue_b32(), sum_f32x);
+    *result = nk_svaddv_f32_(svptrue_b32(), sum_f32x);
 }
 
 #if defined(__clang__)

package/include/numkong/dot/svehalf.h

@@ -33,6 +33,7 @@
 #if NK_TARGET_SVEHALF
 
 #include "numkong/types.h"      // `nk_f16_t`
+#include "numkong/reduce/sve.h" // `nk_svaddv_f64_`
 #include "numkong/dot/serial.h" // `nk_u1x8_popcount_`
 
 #if defined(__cplusplus)
@@ -67,7 +68,7 @@ NK_PUBLIC void nk_dot_f16_svehalf(nk_f16_t const *a_scalars, nk_f16_t const *b_s
 
         idx_scalars += svcnth();
     } while (idx_scalars < count_scalars);
-    *result = svaddv_f32(svptrue_b32(), ab_f32x);
+    *result = nk_svaddv_f32_(svptrue_b32(), ab_f32x);
 }
 
 NK_PUBLIC void nk_dot_f16c_svehalf(nk_f16c_t const *a_pairs, nk_f16c_t const *b_pairs, nk_size_t count_pairs,
@@ -107,8 +108,8 @@ NK_PUBLIC void nk_dot_f16c_svehalf(nk_f16c_t const *a_pairs, nk_f16c_t const *b_
 
         idx_scalars += svcnth();
     } while (idx_scalars < count_pairs);
-    results->real = svaddv_f32(svptrue_b32(), ab_real_f32x);
-    results->imag = svaddv_f32(svptrue_b32(), ab_imag_f32x);
+    results->real = nk_svaddv_f32_(svptrue_b32(), ab_real_f32x);
+    results->imag = nk_svaddv_f32_(svptrue_b32(), ab_imag_f32x);
 }
 
 NK_PUBLIC void nk_vdot_f16c_svehalf(nk_f16c_t const *a_pairs, nk_f16c_t const *b_pairs, nk_size_t count_pairs,
@@ -148,8 +149,8 @@ NK_PUBLIC void nk_vdot_f16c_svehalf(nk_f16c_t const *a_pairs, nk_f16c_t const *b
 
         idx_scalars += svcnth();
     } while (idx_scalars < count_pairs);
-    results->real = svaddv_f32(svptrue_b32(), ab_real_f32x);
-    results->imag = svaddv_f32(svptrue_b32(), ab_imag_f32x);
+    results->real = nk_svaddv_f32_(svptrue_b32(), ab_real_f32x);
+    results->imag = nk_svaddv_f32_(svptrue_b32(), ab_imag_f32x);
 }
 
 #if defined(__clang__)

package/include/numkong/dot/svesdot.h

@@ -34,6 +34,7 @@
 #if NK_TARGET_SVESDOT
 
 #include "numkong/types.h"
+#include "numkong/reduce/sve.h" // `nk_svaddv_f64_`
 
 #if defined(__cplusplus)
 extern "C" {
@@ -57,7 +58,7 @@ NK_PUBLIC void nk_dot_i8_svesdot(nk_i8_t const *a_scalars, nk_i8_t const *b_scal
         sum_i32x = svdot_s32(sum_i32x, a_i8x, b_i8x);
         idx_scalars += svcntb();
     } while (idx_scalars < count_scalars);
-    *result = (nk_i32_t)svaddv_s32(svptrue_b32(), sum_i32x);
+    *result = (nk_i32_t)nk_svaddv_s32_(svptrue_b32(), sum_i32x);
 }
 
 NK_PUBLIC void nk_dot_u8_svesdot(nk_u8_t const *a_scalars, nk_u8_t const *b_scalars, nk_size_t count_scalars,
@@ -71,7 +72,7 @@ NK_PUBLIC void nk_dot_u8_svesdot(nk_u8_t const *a_scalars, nk_u8_t const *b_scal
         sum_u32x = svdot_u32(sum_u32x, a_u8x, b_u8x);
         idx_scalars += svcntb();
     } while (idx_scalars < count_scalars);
-    *result = (nk_u32_t)svaddv_u32(svptrue_b32(), sum_u32x);
+    *result = (nk_u32_t)nk_svaddv_u32_(svptrue_b32(), sum_u32x);
 }
 
 #if defined(__clang__)

package/include/numkong/dots/README.md

@@ -72,6 +72,8 @@ Int8 data is quad-interleaved: [a₀, a₁, a₂, a₃, a₀, a₁, a₂, a₃,
 Tile configuration via `LDTILECFG` sets row counts and column byte-widths per tile — allows undersized tiles at matrix edges without masking.
 Morton Z-curve ordering for tile traversal improves cache reuse when both A and B exceed L2.
 This eliminates the explicit M×N×K loop nesting and register file pressure of vector ISAs — the entire dot-product reduction happens inside the tile instruction.
+FP8 inputs on Sapphire AMX go through an on-the-fly E4M3/E5M2 → BF16 pack via the Ice Lake `VPERMI2W` LUT helpers — port-5-bound but the simplest correct route to feed `TDPBF16PS` tiles.
+Granite Rapids adds `TDPFP16PS` (same tile shape, FP16 operands); the E5M2 variant widens inputs with a single `VPUNPCK*BW` against zero into FP16 tiles at pack time and then reuses the native FP16 compute loop — keeps the intermediate at FP16 precision instead of truncating to BF16 like the Sapphire path.
 
 ### SME Outer-Product Streaming