numkong 7.0.0 → 7.4.1

package/include/numkong/dot/alder.h

@@ -8,10 +8,10 @@
  *
  *  @section dot_alder_instructions AVX-VNNI Instructions Performance
  *
- *      Intrinsic                   Instruction                     Alder Lake  Raptor Lake
- *      _mm256_dpbusd_epi32         VPDPBUSD (YMM, YMM, YMM)        4cy @ p05   4cy @ p05
- *      _mm256_madd_epi16           VPMADDWD (YMM, YMM, YMM)        4cy @ p05   4cy @ p05
- *      _mm256_sad_epu8             VPSADBW (YMM, YMM, YMM)         3cy @ p5    3cy @ p5
+ *      Intrinsic            Instruction               Alder Lake  Raptor Lake
+ *      _mm256_dpbusd_epi32  VPDPBUSD (YMM, YMM, YMM)  4cy @ p05   4cy @ p05
+ *      _mm256_madd_epi16    VPMADDWD (YMM, YMM, YMM)  4cy @ p05   4cy @ p05
+ *      _mm256_sad_epu8      VPSADBW (YMM, YMM, YMM)   3cy @ p5    3cy @ p5
  *
  *  Alder Lake and Raptor Lake support AVX-VNNI (256-bit VNNI)
  *  for accelerated integer dot products. This is the 256-bit variant of AVX-512 VNNI found on Ice Lake.
@@ -208,13 +208,13 @@ NK_INTERNAL void nk_dot_i8x32_finalize_alder(
                                         _mm256_extracti128_si256(state_d->biased_product_sum_i32x8, 1));
 
     // 4-way transpose reduce
-    __m128i t_ab_lo = _mm_unpacklo_epi32(sum_a_i32x4, sum_b_i32x4);
-    __m128i t_cd_lo = _mm_unpacklo_epi32(sum_c_i32x4, sum_d_i32x4);
-    __m128i t_ab_hi = _mm_unpackhi_epi32(sum_a_i32x4, sum_b_i32x4);
-    __m128i t_cd_hi = _mm_unpackhi_epi32(sum_c_i32x4, sum_d_i32x4);
+    __m128i t_ab_low = _mm_unpacklo_epi32(sum_a_i32x4, sum_b_i32x4);
+    __m128i t_cd_low = _mm_unpacklo_epi32(sum_c_i32x4, sum_d_i32x4);
+    __m128i t_ab_high = _mm_unpackhi_epi32(sum_a_i32x4, sum_b_i32x4);
+    __m128i t_cd_high = _mm_unpackhi_epi32(sum_c_i32x4, sum_d_i32x4);
     __m128i biased_i32x4 = _mm_add_epi32(
-        _mm_add_epi32(_mm_unpacklo_epi64(t_ab_lo, t_cd_lo), _mm_unpackhi_epi64(t_ab_lo, t_cd_lo)),
-        _mm_add_epi32(_mm_unpacklo_epi64(t_ab_hi, t_cd_hi), _mm_unpackhi_epi64(t_ab_hi, t_cd_hi)));
+        _mm_add_epi32(_mm_unpacklo_epi64(t_ab_low, t_cd_low), _mm_unpackhi_epi64(t_ab_low, t_cd_low)),
+        _mm_add_epi32(_mm_unpacklo_epi64(t_ab_high, t_cd_high), _mm_unpackhi_epi64(t_ab_high, t_cd_high)));
 
     // Apply compensation: result = biased − 128 × Σb
     __m128i correction_i32x4 = _mm_slli_epi32(b_sums.xmm, 7); // × 128
@@ -328,13 +328,13 @@ NK_INTERNAL void nk_dot_u8x32_finalize_alder(
                                         _mm256_extracti128_si256(state_d->biased_product_sum_i32x8, 1));
 
     // 4-way transpose reduce
-    __m128i t_ab_lo = _mm_unpacklo_epi32(sum_a_i32x4, sum_b_i32x4);
-    __m128i t_cd_lo = _mm_unpacklo_epi32(sum_c_i32x4, sum_d_i32x4);
-    __m128i t_ab_hi = _mm_unpackhi_epi32(sum_a_i32x4, sum_b_i32x4);
-    __m128i t_cd_hi = _mm_unpackhi_epi32(sum_c_i32x4, sum_d_i32x4);
+    __m128i t_ab_low = _mm_unpacklo_epi32(sum_a_i32x4, sum_b_i32x4);
+    __m128i t_cd_low = _mm_unpacklo_epi32(sum_c_i32x4, sum_d_i32x4);
+    __m128i t_ab_high = _mm_unpackhi_epi32(sum_a_i32x4, sum_b_i32x4);
+    __m128i t_cd_high = _mm_unpackhi_epi32(sum_c_i32x4, sum_d_i32x4);
     __m128i biased_i32x4 = _mm_add_epi32(
-        _mm_add_epi32(_mm_unpacklo_epi64(t_ab_lo, t_cd_lo), _mm_unpackhi_epi64(t_ab_lo, t_cd_lo)),
-        _mm_add_epi32(_mm_unpacklo_epi64(t_ab_hi, t_cd_hi), _mm_unpackhi_epi64(t_ab_hi, t_cd_hi)));
+        _mm_add_epi32(_mm_unpacklo_epi64(t_ab_low, t_cd_low), _mm_unpackhi_epi64(t_ab_low, t_cd_low)),
+        _mm_add_epi32(_mm_unpacklo_epi64(t_ab_high, t_cd_high), _mm_unpackhi_epi64(t_ab_high, t_cd_high)));
 
     // Apply compensation: result = biased + 128 × Σb
     __m128i correction_i32x4 = _mm_slli_epi32(b_sums.xmm, 7); // × 128
@@ -355,20 +355,20 @@ NK_INTERNAL void nk_sum_i8x32_init_alder(nk_sum_i8x32_state_alder_t *state) {
     state->biased_sum_u64x4 = _mm256_setzero_si256();
 }
 NK_INTERNAL void nk_sum_i8x32_update_alder(nk_sum_i8x32_state_alder_t *state, nk_b256_vec_t vector) {
-    /* Convert signed→unsigned via XOR 0x80, then SAD against zero gives sum of unsigned values */
+    // Convert signed→unsigned via XOR 0x80, then SAD against zero gives sum of unsigned values
     __m256i vector_unsigned_u8x32 = _mm256_xor_si256(vector.ymm, _mm256_set1_epi8((char)0x80));
     __m256i sad_result_u64x4 = _mm256_sad_epu8(vector_unsigned_u8x32, _mm256_setzero_si256());
     state->biased_sum_u64x4 = _mm256_add_epi64(state->biased_sum_u64x4, sad_result_u64x4);
 }
 NK_INTERNAL nk_i32_t nk_sum_i8x32_finalize_alder(nk_sum_i8x32_state_alder_t const *state, nk_size_t count) {
-    /* Horizontal reduce u64x4 → scalar */
+    // Horizontal reduce u64x4 → scalar
     __m128i low_u64x2 = _mm256_castsi256_si128(state->biased_sum_u64x4);
     __m128i high_u64x2 = _mm256_extracti128_si256(state->biased_sum_u64x4, 1);
     __m128i paired_u64x2 = _mm_add_epi64(low_u64x2, high_u64x2);
     __m128i shuffled_u64x2 = _mm_shuffle_epi32(paired_u64x2, _MM_SHUFFLE(1, 0, 3, 2));
     __m128i total_u64x2 = _mm_add_epi64(paired_u64x2, shuffled_u64x2);
     nk_u64_t unsigned_sum = (nk_u64_t)_mm_cvtsi128_si64(total_u64x2);
-    /* Undo XOR bias: signed_sum = unsigned_sum - 128 * count */
+    // Undo XOR bias: signed_sum = unsigned_sum - 128 * count
     return (nk_i32_t)((nk_i64_t)unsigned_sum - 128 * (nk_i64_t)count);
 }
 
@@ -403,10 +403,10 @@ NK_PUBLIC void nk_dot_e2m3_alder(nk_e2m3_t const *a_scalars, nk_e2m3_t const *b_
     // This is the Alder Lake (256-bit AVX-VNNI) variant of the Ice Lake kernel.
     // DPBUSD replaces MADDUBS+MADD (2 instructions → 1), accumulating u8×i8→i32 directly.
     //
-    __m256i const lut_lower_u8x32 = _mm256_set_epi8(               //
+    __m256i const lut_low_u8x32 = _mm256_set_epi8(                 //
         30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, //
         30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0);
-    __m256i const lut_upper_u8x32 = _mm256_set_epi8(                       //
+    __m256i const lut_high_u8x32 = _mm256_set_epi8(                        //
         120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32, //
         120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32);
     __m256i const nibble_mask_u8x32 = _mm256_set1_epi8(0x0F);
@@ -436,18 +436,18 @@ nk_dot_e2m3_alder_cycle:
     __m256i b_magnitude_u8x32 = _mm256_and_si256(b_e2m3_u8x32, magnitude_mask_u8x32);
     __m256i a_shuffle_index_u8x32 = _mm256_and_si256(a_magnitude_u8x32, nibble_mask_u8x32);
     __m256i b_shuffle_index_u8x32 = _mm256_and_si256(b_magnitude_u8x32, nibble_mask_u8x32);
-    __m256i a_upper_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(a_magnitude_u8x32, half_select_u8x32),
-                                                     half_select_u8x32);
-    __m256i b_upper_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(b_magnitude_u8x32, half_select_u8x32),
-                                                     half_select_u8x32);
+    __m256i a_high_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(a_magnitude_u8x32, half_select_u8x32),
+                                                    half_select_u8x32);
+    __m256i b_high_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(b_magnitude_u8x32, half_select_u8x32),
+                                                    half_select_u8x32);
 
     // Dual VPSHUFB: lookup in both halves, blend based on bit 4
-    __m256i a_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_lower_u8x32, a_shuffle_index_u8x32),
-                                                  _mm256_shuffle_epi8(lut_upper_u8x32, a_shuffle_index_u8x32),
-                                                  a_upper_select_u8x32);
-    __m256i b_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_lower_u8x32, b_shuffle_index_u8x32),
-                                                  _mm256_shuffle_epi8(lut_upper_u8x32, b_shuffle_index_u8x32),
-                                                  b_upper_select_u8x32);
+    __m256i a_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_low_u8x32, a_shuffle_index_u8x32),
+                                                  _mm256_shuffle_epi8(lut_high_u8x32, a_shuffle_index_u8x32),
+                                                  a_high_select_u8x32);
+    __m256i b_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_low_u8x32, b_shuffle_index_u8x32),
+                                                  _mm256_shuffle_epi8(lut_high_u8x32, b_shuffle_index_u8x32),
+                                                  b_high_select_u8x32);
 
     // Combined sign: (a ^ b) & 0x20, negate b where signs differ
     __m256i sign_combined_u8x32 = _mm256_and_si256(_mm256_xor_si256(a_e2m3_u8x32, b_e2m3_u8x32), sign_mask_u8x32);
@@ -474,10 +474,10 @@ NK_INTERNAL void nk_dot_e2m3x32_update_alder(nk_dot_e2m3x32_state_alder_t *state
                                              nk_size_t depth_offset, nk_size_t active_dimensions) {
     nk_unused_(depth_offset);
     nk_unused_(active_dimensions);
-    __m256i const lut_lower_u8x32 = _mm256_set_epi8(               //
+    __m256i const lut_low_u8x32 = _mm256_set_epi8(                 //
         30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0, //
         30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0);
-    __m256i const lut_upper_u8x32 = _mm256_set_epi8(                       //
+    __m256i const lut_high_u8x32 = _mm256_set_epi8(                        //
         120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32, //
         120, 112, 104, 96, 88, 80, 72, 64, 60, 56, 52, 48, 44, 40, 36, 32);
     __m256i const nibble_mask_u8x32 = _mm256_set1_epi8(0x0F);
@@ -493,18 +493,18 @@ NK_INTERNAL void nk_dot_e2m3x32_update_alder(nk_dot_e2m3x32_state_alder_t *state
     __m256i b_magnitude_u8x32 = _mm256_and_si256(b_e2m3_u8x32, magnitude_mask_u8x32);
     __m256i a_shuffle_index_u8x32 = _mm256_and_si256(a_magnitude_u8x32, nibble_mask_u8x32);
     __m256i b_shuffle_index_u8x32 = _mm256_and_si256(b_magnitude_u8x32, nibble_mask_u8x32);
-    __m256i a_upper_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(a_magnitude_u8x32, half_select_u8x32),
-                                                     half_select_u8x32);
-    __m256i b_upper_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(b_magnitude_u8x32, half_select_u8x32),
-                                                     half_select_u8x32);
+    __m256i a_high_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(a_magnitude_u8x32, half_select_u8x32),
+                                                    half_select_u8x32);
+    __m256i b_high_select_u8x32 = _mm256_cmpeq_epi8(_mm256_and_si256(b_magnitude_u8x32, half_select_u8x32),
+                                                    half_select_u8x32);
 
     // Dual VPSHUFB + blend
-    __m256i a_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_lower_u8x32, a_shuffle_index_u8x32),
-                                                  _mm256_shuffle_epi8(lut_upper_u8x32, a_shuffle_index_u8x32),
-                                                  a_upper_select_u8x32);
-    __m256i b_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_lower_u8x32, b_shuffle_index_u8x32),
-                                                  _mm256_shuffle_epi8(lut_upper_u8x32, b_shuffle_index_u8x32),
-                                                  b_upper_select_u8x32);
+    __m256i a_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_low_u8x32, a_shuffle_index_u8x32),
+                                                  _mm256_shuffle_epi8(lut_high_u8x32, a_shuffle_index_u8x32),
+                                                  a_high_select_u8x32);
+    __m256i b_unsigned_u8x32 = _mm256_blendv_epi8(_mm256_shuffle_epi8(lut_low_u8x32, b_shuffle_index_u8x32),
+                                                  _mm256_shuffle_epi8(lut_high_u8x32, b_shuffle_index_u8x32),
+                                                  b_high_select_u8x32);
 
     // Combined sign + conditional negate
     __m256i sign_combined_u8x32 = _mm256_and_si256(_mm256_xor_si256(a_e2m3_u8x32, b_e2m3_u8x32), sign_mask_u8x32);

package/include/numkong/dot/diamond.h

@@ -0,0 +1,158 @@
+/**
+ *  @brief SIMD-accelerated Dot Products for Diamond Rapids.
+ *  @file include/numkong/dot/diamond.h
+ *  @author Ash Vardanian
+ *  @date March 23, 2026
+ *
+ *  @sa include/numkong/dot.h
+ *
+ *  @section dot_diamond_instructions Key AVX10.2 FP8 + FP16 VNNI Instructions
+ *
+ *      Intrinsic            Instruction                   Diamond Rapids
+ *      _mm512_cvthf8_ph     VCVTHF82PH (ZMM, YMM)         ~3cy (estimated)
+ *      _mm512_cvtbf8_ph     VCVTBF82PH (ZMM, YMM)         ~3cy (estimated)
+ *      _mm512_dpph_ps       VDPPHPS (ZMM, ZMM, ZMM)       ~6cy (estimated)
+ *
+ *  Diamond Rapids (AVX10.2) introduces native FP8→FP16 conversion via VCVTHF82PH (E4M3→FP16)
+ *  and VCVTBF82PH (E5M2→FP16), replacing the multi-instruction arithmetic conversion used by
+ *  Genoa's BF16 path. VDPPHPS then computes two FP16 dot products per 32-bit lane, accumulating
+ *  into FP32 — providing the same 32-element throughput as Genoa's VDPBF16PS but with FP16
+ *  intermediate precision (10-bit mantissa vs BF16's 7-bit).
+ *
+ *  @section dot_diamond_stateful Stateful Streaming Logic
+ *
+ *  Defines stateful init/update/finalize helpers for tiled GEMM via the dots/ macros:
+ *  - nk_dot_through_f16_state_diamond_t_ shared by both E4M3 and E5M2 (FP16→VDPPHPS→FP32)
+ */
+#ifndef NK_DOT_DIAMOND_H
+#define NK_DOT_DIAMOND_H
+
+#if NK_TARGET_X86_
+#if NK_TARGET_DIAMOND
+
+#include "numkong/types.h"
+#include "numkong/cast/diamond.h"   // `nk_load_e4m3x32_to_f16x32_diamond_`
+#include "numkong/reduce/skylake.h" // `nk_reduce_add_f32x16_skylake_`
+#include "numkong/dot/skylake.h"    // `nk_dot_through_f32_finalize_skylake_`
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(                                                                                    \
+    __attribute__((target("avx2,avx512f,avx512vl,avx512bw,avx512dq,avx512fp16,avx10.2-512,f16c,fma,bmi,bmi2"))), \
+    apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "avx512dq", "avx512fp16", "avx10.2-512", "f16c", "fma", \
+                   "bmi", "bmi2")
+#endif
+
+NK_PUBLIC void nk_dot_e4m3_diamond(nk_e4m3_t const *a_scalars, nk_e4m3_t const *b_scalars, nk_size_t count_scalars,
+                                   nk_f32_t *result) {
+    __m256i a_e4m3x32, b_e4m3x32;
+    __m512 sum_f32x16 = _mm512_setzero_ps();
+
+nk_dot_e4m3_diamond_cycle:
+    if (count_scalars < 32) {
+        __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, count_scalars);
+        a_e4m3x32 = _mm256_maskz_loadu_epi8(mask, a_scalars);
+        b_e4m3x32 = _mm256_maskz_loadu_epi8(mask, b_scalars);
+        count_scalars = 0;
+    }
+    else {
+        a_e4m3x32 = _mm256_loadu_epi8(a_scalars);
+        b_e4m3x32 = _mm256_loadu_epi8(b_scalars);
+        a_scalars += 32, b_scalars += 32, count_scalars -= 32;
+    }
+    __m512h a_f16x32 = _mm512_cvthf8_ph(a_e4m3x32);
+    __m512h b_f16x32 = _mm512_cvthf8_ph(b_e4m3x32);
+    sum_f32x16 = _mm512_dpph_ps(sum_f32x16, a_f16x32, b_f16x32);
+    if (count_scalars) goto nk_dot_e4m3_diamond_cycle;
+
+    *result = nk_reduce_add_f32x16_skylake_(sum_f32x16);
+}
+
+NK_PUBLIC void nk_dot_e5m2_diamond(nk_e5m2_t const *a_scalars, nk_e5m2_t const *b_scalars, nk_size_t count_scalars,
+                                   nk_f32_t *result) {
+    __m256i a_e5m2x32, b_e5m2x32;
+    __m512 sum_f32x16 = _mm512_setzero_ps();
+
+nk_dot_e5m2_diamond_cycle:
+    if (count_scalars < 32) {
+        __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, count_scalars);
+        a_e5m2x32 = _mm256_maskz_loadu_epi8(mask, a_scalars);
+        b_e5m2x32 = _mm256_maskz_loadu_epi8(mask, b_scalars);
+        count_scalars = 0;
+    }
+    else {
+        a_e5m2x32 = _mm256_loadu_epi8(a_scalars);
+        b_e5m2x32 = _mm256_loadu_epi8(b_scalars);
+        a_scalars += 32, b_scalars += 32, count_scalars -= 32;
+    }
+    __m512h a_f16x32 = _mm512_cvtbf8_ph(a_e5m2x32);
+    __m512h b_f16x32 = _mm512_cvtbf8_ph(b_e5m2x32);
+    sum_f32x16 = _mm512_dpph_ps(sum_f32x16, a_f16x32, b_f16x32);
+    if (count_scalars) goto nk_dot_e5m2_diamond_cycle;
+
+    *result = nk_reduce_add_f32x16_skylake_(sum_f32x16);
+}
+
+NK_PUBLIC void nk_dot_f16_diamond(nk_f16_t const *a_scalars, nk_f16_t const *b_scalars, nk_size_t count_scalars,
+                                  nk_f32_t *result) {
+    __m512h a_f16x32, b_f16x32;
+    __m512 sum_f32x16 = _mm512_setzero_ps();
+
+nk_dot_f16_diamond_cycle:
+    if (count_scalars < 32) {
+        __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, count_scalars);
+        a_f16x32 = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, a_scalars));
+        b_f16x32 = _mm512_castsi512_ph(_mm512_maskz_loadu_epi16(mask, b_scalars));
+        count_scalars = 0;
+    }
+    else {
+        a_f16x32 = _mm512_castsi512_ph(_mm512_loadu_epi16(a_scalars));
+        b_f16x32 = _mm512_castsi512_ph(_mm512_loadu_epi16(b_scalars));
+        a_scalars += 32, b_scalars += 32, count_scalars -= 32;
+    }
+    sum_f32x16 = _mm512_dpph_ps(sum_f32x16, a_f16x32, b_f16x32);
+    if (count_scalars) goto nk_dot_f16_diamond_cycle;
+
+    *result = nk_reduce_add_f32x16_skylake_(sum_f32x16);
+}
+
+typedef nk_dot_through_f32_state_skylake_t_ nk_dot_through_f16_state_diamond_t_;
+
+NK_INTERNAL void nk_dot_through_f16_init_diamond_(nk_dot_through_f16_state_diamond_t_ *state) {
+    state->sum_f32x16 = _mm512_setzero();
+}
+
+NK_INTERNAL void nk_dot_through_f16_update_diamond_(nk_dot_through_f16_state_diamond_t_ *state, nk_b512_vec_t a,
+                                                    nk_b512_vec_t b, nk_size_t depth_offset,
+                                                    nk_size_t active_dimensions) {
+    nk_unused_(depth_offset);
+    nk_unused_(active_dimensions);
+    state->sum_f32x16 = _mm512_dpph_ps(state->sum_f32x16, a.zmm_ph, b.zmm_ph);
+}
+
+NK_INTERNAL void nk_dot_through_f16_finalize_diamond_(                                                      //
+    nk_dot_through_f16_state_diamond_t_ const *state_a, nk_dot_through_f16_state_diamond_t_ const *state_b, //
+    nk_dot_through_f16_state_diamond_t_ const *state_c, nk_dot_through_f16_state_diamond_t_ const *state_d, //
+    nk_size_t total_dimensions, nk_b128_vec_t *result) {
+    nk_dot_through_f32_finalize_skylake_(state_a, state_b, state_c, state_d, total_dimensions, result);
+}
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_DIAMOND
+#endif // NK_TARGET_X86_
+#endif // NK_DOT_DIAMOND_H

package/include/numkong/dot/genoa.h

@@ -8,10 +8,10 @@
  *
  *  @section dot_genoa_instructions Key AVX-512 BF16 Instructions
  *
- *      Intrinsic                   Instruction                     Latency     Throughput  Ports
- *      _mm512_dpbf16_ps            VDPBF16PS (ZMM, ZMM, ZMM)       4cy         0.5/cy      p01
- *      _mm512_fmadd_ps             VFMADD132PS (ZMM, ZMM, ZMM)     4cy         0.5/cy      p01
- *      _mm512_add_ps               VADDPS (ZMM, ZMM, ZMM)          4cy         0.5/cy      p01
+ *      Intrinsic         Instruction                  Genoa      Alder Lake
+ *      _mm512_dpbf16_ps  VDPBF16PS (ZMM, ZMM, ZMM)    6cy @ p01  8cy @ p0+p0+p5+p5
+ *      _mm512_fmadd_ps   VFMADD132PS (ZMM, ZMM, ZMM)  4cy @ p01  4cy @ p0
+ *      _mm512_add_ps     VADDPS (ZMM, ZMM, ZMM)       4cy @ p01  3cy @ p05
  *
  *  AMD Genoa introduces native AVX-512 BF16 support with VDPBF16PS, which computes two BF16 dot products
  *  per 32-bit lane (32 BF16 multiplies accumulated into 16 FP32 values per instruction). This provides
@@ -208,32 +208,6 @@ nk_vdot_bf16c_genoa_cycle:
     result->imag = nk_reduce_add_f32x16_skylake_(sum_imag_f32x16);
 }
 
-NK_PUBLIC void nk_dot_e4m3_genoa(nk_e4m3_t const *a_scalars, nk_e4m3_t const *b_scalars, nk_size_t count_scalars,
-                                 nk_f32_t *result) {
-    __m256i a_e4m3x32, b_e4m3x32;
-    __m512 sum_f32x16 = _mm512_setzero_ps();
-
-nk_dot_e4m3_genoa_cycle:
-    if (count_scalars < 32) {
-        __mmask32 mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, count_scalars);
-        a_e4m3x32 = _mm256_maskz_loadu_epi8(mask, a_scalars);
-        b_e4m3x32 = _mm256_maskz_loadu_epi8(mask, b_scalars);
-        count_scalars = 0;
-    }
-    else {
-        a_e4m3x32 = _mm256_loadu_epi8(a_scalars);
-        b_e4m3x32 = _mm256_loadu_epi8(b_scalars);
-        a_scalars += 32, b_scalars += 32, count_scalars -= 32;
-    }
-    // Convert E4M3 to BF16 and compute dot product
-    __m512i a_bf16x32 = nk_e4m3x32_to_bf16x32_icelake_(a_e4m3x32);
-    __m512i b_bf16x32 = nk_e4m3x32_to_bf16x32_icelake_(b_e4m3x32);
-    sum_f32x16 = _mm512_dpbf16_ps(sum_f32x16, nk_m512bh_from_m512i_(a_bf16x32), nk_m512bh_from_m512i_(b_bf16x32));
-    if (count_scalars) goto nk_dot_e4m3_genoa_cycle;
-
-    *result = nk_reduce_add_f32x16_skylake_(sum_f32x16);
-}
-
 NK_PUBLIC void nk_dot_e5m2_genoa(nk_e5m2_t const *a_scalars, nk_e5m2_t const *b_scalars, nk_size_t count_scalars,
                                  nk_f32_t *result) {
     __m256i a_e5m2x32, b_e5m2x32;