numkong 7.0.0 → 7.4.2

package/include/numkong/dots/alder.h

@@ -34,9 +34,10 @@ extern "C" {
 nk_define_cross_compensated_pack_size_(dots, i8, alder, i8, i8,
                                        /*sum_value_type=*/i32, /*norm_value_type=*/u32,
                                        /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
-nk_define_cross_compensated_pack_(dots, i8, alder, i8, i8, nk_assign_from_to_,
-                                  /*sum_value_type=*/i32, /*norm_value_type=*/u32, nk_dots_reduce_moments_i8_,
-                                  /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
+nk_define_cross_compensated_pack_(dots, i8, alder, i8, i8, nk_b128_vec_t, nk_load_b128_haswell_,
+                                  nk_partial_load_b8x16_serial_, nk_store_b128_haswell_, nk_partial_store_b8x16_serial_,
+                                  /*simd_width=*/16, /*sum_value_type=*/i32, /*norm_value_type=*/u32,
+                                  nk_dots_reduce_moments_i8_, /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
 nk_define_cross_compensated_symmetric_(dots, i8, alder, i8, i32,
                                        /*sum_value_type=*/i32, /*norm_value_type=*/u32, nk_b256_vec_t,
                                        nk_dot_i8x32_state_alder_t, nk_b128_vec_t, nk_dot_i8x32_init_alder,
@@ -60,9 +61,10 @@ nk_define_cross_compensated_packed_(dots, i8, alder, i8, i8, i32,
 nk_define_cross_compensated_pack_size_(dots, u8, alder, u8, u8,
                                        /*sum_value_type=*/u32, /*norm_value_type=*/u32,
                                        /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
-nk_define_cross_compensated_pack_(dots, u8, alder, u8, u8, nk_assign_from_to_,
-                                  /*sum_value_type=*/u32, /*norm_value_type=*/u32, nk_dots_reduce_moments_u8_,
-                                  /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
+nk_define_cross_compensated_pack_(dots, u8, alder, u8, u8, nk_b128_vec_t, nk_load_b128_haswell_,
+                                  nk_partial_load_b8x16_serial_, nk_store_b128_haswell_, nk_partial_store_b8x16_serial_,
+                                  /*simd_width=*/16, /*sum_value_type=*/u32, /*norm_value_type=*/u32,
+                                  nk_dots_reduce_moments_u8_, /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
 nk_define_cross_compensated_symmetric_(dots, u8, alder, u8, u32,
                                        /*sum_value_type=*/u32, /*norm_value_type=*/u32, nk_b256_vec_t,
                                        nk_dot_u8x32_state_alder_t, nk_b128_vec_t, nk_dot_u8x32_init_alder,
@@ -85,9 +87,10 @@ nk_define_cross_compensated_packed_(dots, u8, alder, u8, u8, u32,
 /* E2M3 GEMM via DPBUSD integer path: depth_simd_dimensions=32 (32 e2m3s = 32 bytes = AVX2 register width) */
 nk_define_cross_pack_size_(dots, e2m3, alder, e2m3, e2m3, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/32,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, e2m3, alder, e2m3, e2m3, nk_assign_from_to_, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_e2m3_, /*depth_simd_dimensions=*/32,
-                      /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, e2m3, alder, e2m3, e2m3, nk_b256_vec_t, nk_load_b256_haswell_,
+                      nk_partial_load_b8x32_serial_, nk_store_b256_haswell_, nk_partial_store_b8x32_serial_,
+                      /*simd_width=*/32, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e2m3_,
+                      /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, e2m3, alder, e2m3, f32, nk_b256_vec_t, nk_dot_e2m3x32_state_alder_t, nk_b128_vec_t,
                            nk_dot_e2m3x32_init_alder, nk_load_b256_haswell_, nk_partial_load_b8x32_serial_,
                            nk_dot_e2m3x32_update_alder, nk_dot_e2m3x32_finalize_alder, nk_store_b128_haswell_,

package/include/numkong/dots/diamond.h

@@ -0,0 +1,86 @@
+/**
+ *  @brief SIMD-accelerated Batched Dot Products for Diamond Rapids.
+ *  @file include/numkong/dots/diamond.h
+ *  @author Ash Vardanian
+ *  @date March 23, 2026
+ *
+ *  @sa include/numkong/dots.h
+ *
+ *  Uses VCVTHF82PH/VCVTBF82PH for native FP8→FP16 conversion, then VDPPHPS for
+ *  FP16-pair dot products accumulating into FP32. Processes 32 FP8 elements per iteration.
+ */
+#ifndef NK_DOTS_DIAMOND_H
+#define NK_DOTS_DIAMOND_H
+
+#if NK_TARGET_X86_
+#if NK_TARGET_DIAMOND
+
+#include "numkong/dot/diamond.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(                                                                                    \
+    __attribute__((target("avx2,avx512f,avx512vl,avx512bw,avx512dq,avx512fp16,avx10.2-512,f16c,fma,bmi,bmi2"))), \
+    apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "avx512dq", "avx512fp16", "avx10.2-512", "f16c", "fma", \
+                   "bmi", "bmi2")
+#endif
+
+/* E4M3 GEMM: depth_simd_dimensions=32 (32 e4m3s = 32 bytes), FP16 intermediate, F32 accumulator */
+nk_define_cross_pack_size_(dots, e4m3, diamond, e4m3, e4m3, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/32,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, e4m3, diamond, e4m3, e4m3, nk_b512_vec_t, nk_load_b512_skylake_,
+                      nk_partial_load_b8x64_skylake_, nk_store_b512_skylake_, nk_partial_store_b8x64_skylake_,
+                      /*simd_width=*/64, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e4m3_,
+                      /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, e4m3, diamond, e4m3, f32, nk_b512_vec_t, nk_dot_through_f16_state_diamond_t_,
+                           nk_b128_vec_t, nk_dot_through_f16_init_diamond_, nk_load_e4m3x32_to_f16x32_diamond_,
+                           nk_partial_load_e4m3x32_to_f16x32_diamond_, nk_dot_through_f16_update_diamond_,
+                           nk_dot_through_f16_finalize_diamond_, nk_store_b128_haswell_,
+                           nk_partial_store_b32x4_skylake_,
+                           /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, e4m3, diamond, e4m3, e4m3, f32, nk_b512_vec_t, nk_dot_through_f16_state_diamond_t_,
+                        nk_b128_vec_t, nk_dot_through_f16_init_diamond_, nk_load_e4m3x32_to_f16x32_diamond_,
+                        nk_partial_load_e4m3x32_to_f16x32_diamond_, nk_load_e4m3x32_to_f16x32_diamond_,
+                        nk_partial_load_e4m3x32_to_f16x32_diamond_, nk_dot_through_f16_update_diamond_,
+                        nk_dot_through_f16_finalize_diamond_, nk_store_b128_haswell_, nk_partial_store_b32x4_skylake_,
+                        /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
+
+/* E5M2 GEMM: depth_simd_dimensions=32 (32 e5m2s = 32 bytes), FP16 intermediate, F32 accumulator */
+nk_define_cross_pack_size_(dots, e5m2, diamond, e5m2, e5m2, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/32,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, e5m2, diamond, e5m2, e5m2, nk_b512_vec_t, nk_load_b512_skylake_,
+                      nk_partial_load_b8x64_skylake_, nk_store_b512_skylake_, nk_partial_store_b8x64_skylake_,
+                      /*simd_width=*/64, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e5m2_,
+                      /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, e5m2, diamond, e5m2, f32, nk_b512_vec_t, nk_dot_through_f16_state_diamond_t_,
+                           nk_b128_vec_t, nk_dot_through_f16_init_diamond_, nk_load_e5m2x32_to_f16x32_diamond_,
+                           nk_partial_load_e5m2x32_to_f16x32_diamond_, nk_dot_through_f16_update_diamond_,
+                           nk_dot_through_f16_finalize_diamond_, nk_store_b128_haswell_,
+                           nk_partial_store_b32x4_skylake_,
+                           /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, e5m2, diamond, e5m2, e5m2, f32, nk_b512_vec_t, nk_dot_through_f16_state_diamond_t_,
+                        nk_b128_vec_t, nk_dot_through_f16_init_diamond_, nk_load_e5m2x32_to_f16x32_diamond_,
+                        nk_partial_load_e5m2x32_to_f16x32_diamond_, nk_load_e5m2x32_to_f16x32_diamond_,
+                        nk_partial_load_e5m2x32_to_f16x32_diamond_, nk_dot_through_f16_update_diamond_,
+                        nk_dot_through_f16_finalize_diamond_, nk_store_b128_haswell_, nk_partial_store_b32x4_skylake_,
+                        /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_DIAMOND
+#endif // NK_TARGET_X86_
+#endif // NK_DOTS_DIAMOND_H

package/include/numkong/dots/genoa.h

@@ -31,8 +31,10 @@ extern "C" {
 /* BF16 GEMM: depth_simd_dimensions=32 (32 bf16s = 64 bytes = 1 cache line) */
 nk_define_cross_pack_size_(dots, bf16, genoa, bf16, bf16, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/32,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, bf16, genoa, bf16, bf16, nk_assign_from_to_, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_bf16_, /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, bf16, genoa, bf16, bf16, nk_b512_vec_t, nk_load_b512_skylake_,
+                      nk_partial_load_b16x32_skylake_, nk_store_b512_skylake_, nk_partial_store_b16x32_skylake_,
+                      /*simd_width=*/32, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_bf16_,
+                      /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, bf16, genoa, bf16, f32, nk_b512_vec_t, nk_dot_through_bf16_state_genoa_t_,
                            nk_b128_vec_t, nk_dot_through_bf16_init_genoa_, nk_load_b512_skylake_,
                            nk_partial_load_b16x32_skylake_, nk_dot_through_bf16_update_genoa_,
@@ -48,7 +50,9 @@ nk_define_cross_packed_(dots, bf16, genoa, bf16, bf16, f32, nk_b512_vec_t, nk_do
 /* E4M3 GEMM: depth_simd_dimensions=32 (32 e4m3s = 32 bytes = half cache line), F32 accumulator */
 nk_define_cross_pack_size_(dots, e4m3, genoa, e4m3, bf16, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/32,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, e4m3, genoa, e4m3, bf16, nk_e4m3_to_bf16, /*norm_value_type=*/f32,
+nk_define_cross_pack_(dots, e4m3, genoa, e4m3, bf16, nk_b512_vec_t, nk_load_e4m3x32_to_bf16x32_icelake_,
+                      nk_partial_load_e4m3x32_to_bf16x32_icelake_, nk_store_b512_skylake_,
+                      nk_partial_store_b16x32_skylake_, /*simd_width=*/32, /*norm_value_type=*/f32,
                       nk_dots_reduce_sumsq_e4m3_, /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, e4m3, genoa, e4m3, f32, nk_b512_vec_t, nk_dot_through_bf16_state_genoa_t_,
                            nk_b128_vec_t, nk_dot_through_bf16_init_genoa_, nk_load_e4m3x32_to_bf16x32_icelake_,
@@ -65,7 +69,9 @@ nk_define_cross_packed_(dots, e4m3, genoa, e4m3, bf16, f32, nk_b512_vec_t, nk_do
 /* E5M2 GEMM: depth_simd_dimensions=32 (32 e5m2s = 32 bytes = half cache line), F32 accumulator */
 nk_define_cross_pack_size_(dots, e5m2, genoa, e5m2, bf16, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/32,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, e5m2, genoa, e5m2, bf16, nk_e5m2_to_bf16, /*norm_value_type=*/f32,
+nk_define_cross_pack_(dots, e5m2, genoa, e5m2, bf16, nk_b512_vec_t, nk_load_e5m2x32_to_bf16x32_icelake_,
+                      nk_partial_load_e5m2x32_to_bf16x32_icelake_, nk_store_b512_skylake_,
+                      nk_partial_store_b16x32_skylake_, /*simd_width=*/32, /*norm_value_type=*/f32,
                       nk_dots_reduce_sumsq_e5m2_, /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, e5m2, genoa, e5m2, f32, nk_b512_vec_t, nk_dot_through_bf16_state_genoa_t_,
                            nk_b128_vec_t, nk_dot_through_bf16_init_genoa_, nk_load_e5m2x32_to_bf16x32_icelake_,

package/include/numkong/dots/haswell.h

@@ -8,12 +8,12 @@
  *
  *  @section haswell_dots_instructions Key AVX2/FMA GEMM Instructions
  *
- *      Intrinsic                   Instruction                     Latency     Throughput  Ports
- *      _mm256_fmadd_ps/pd          VFMADD (YMM, YMM, YMM)          5cy         0.5/cy      p01
- *      _mm256_mul_ps               VMULPS (YMM, YMM, YMM)          5cy         0.5/cy      p01
- *      _mm256_add_ps               VADDPS (YMM, YMM, YMM)          3cy         1/cy        p01
- *      _mm256_cvtph_ps             VCVTPH2PS (YMM, XMM)            5cy         1/cy        p01
- *      _mm256_madd_epi16           VPMADDWD (YMM, YMM, YMM)        5cy         1/cy        p0
+ *      Intrinsic           Instruction               Haswell    Genoa
+ *      _mm256_fmadd_ps/pd  VFMADD (YMM, YMM, YMM)    5cy @ p01  4cy @ p01
+ *      _mm256_mul_ps       VMULPS (YMM, YMM, YMM)    5cy @ p01  3cy @ p01
+ *      _mm256_add_ps       VADDPS (YMM, YMM, YMM)    3cy @ p01  3cy @ p23
+ *      _mm256_cvtph_ps     VCVTPH2PS (YMM, XMM)      5cy @ p01  4cy @ p12+p23
+ *      _mm256_madd_epi16   VPMADDWD (YMM, YMM, YMM)  5cy @ p01  3cy @ p01
  *
  *  GEMM kernels use tiled dot products with 4-way parallel accumulation to hide FMA latency.
  *  Type-specific tile sizes: f32/f64 use depth_simd_dimensions=4, f16/bf16 use depth_simd_dimensions=8,
@@ -43,9 +43,9 @@ extern "C" {
 /* F32 GEMM: depth_simd_dimensions=4 (4 f32s = 16 bytes for f32->f64 upcast accumulation) */
 nk_define_cross_pack_size_(dots, f32, haswell, f32, f32, /*norm_value_type=*/f64, /*depth_simd_dimensions=*/4,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, f32, haswell, f32, f32, nk_assign_from_to_, /*norm_value_type=*/f64,
-                      nk_dots_reduce_sumsq_f32_,
-                      /*depth_simd_dimensions=*/4, /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, f32, haswell, f32, f32, nk_b256_vec_t, nk_load_b256_haswell_, nk_partial_load_b32x8_serial_,
+                      nk_store_b256_haswell_, nk_partial_store_b32x8_serial_, /*simd_width=*/8, /*norm_value_type=*/f64,
+                      nk_dots_reduce_sumsq_f32_, /*depth_simd_dimensions=*/4, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, f32, haswell, f32, f64, nk_b128_vec_t, nk_dot_f32x4_state_haswell_t, nk_b256_vec_t,
                            nk_dot_f32x4_init_haswell, nk_load_b128_haswell_, nk_partial_load_b32x4_haswell_,
                            nk_dot_f32x4_update_haswell, nk_dot_f32x4_finalize_haswell, nk_store_b256_haswell_,
@@ -60,9 +60,10 @@ nk_define_cross_packed_(dots, f32, haswell, f32, f32, f64, nk_b128_vec_t, nk_dot
 /* F64 GEMM: depth_simd_dimensions=4 (4 f64s = 32 bytes = AVX2 register width) */
 nk_define_cross_pack_size_(dots, f64, haswell, f64, f64, /*norm_value_type=*/f64, /*depth_simd_dimensions=*/4,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, f64, haswell, f64, f64, nk_assign_from_to_, /*norm_value_type=*/f64,
-                      nk_dots_reduce_sumsq_f64_,
-                      /*depth_simd_dimensions=*/4, /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, f64, haswell, f64, f64, nk_b256_vec_t, nk_load_b256_haswell_,
+                      nk_partial_load_b64x4_haswell_, nk_store_b256_haswell_, nk_partial_store_b64x4_haswell_,
+                      /*simd_width=*/4, /*norm_value_type=*/f64, nk_dots_reduce_sumsq_f64_, /*depth_simd_dimensions=*/4,
+                      /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, f64, haswell, f64, f64, nk_b256_vec_t, nk_dot_f64x4_state_haswell_t, nk_b256_vec_t,
                            nk_dot_f64x4_init_haswell, nk_load_b256_haswell_, nk_partial_load_b64x4_haswell_,
                            nk_dot_f64x4_update_haswell, nk_dot_f64x4_finalize_haswell, nk_store_b256_haswell_,
@@ -77,8 +78,10 @@ nk_define_cross_packed_(dots, f64, haswell, f64, f64, f64, nk_b256_vec_t, nk_dot
 /* F16 GEMM: depth_simd_dimensions=8 (8 f16s = 16 bytes = 128-bit input) → upcasted to 8×f32 (256-bit) */
 nk_define_cross_pack_size_(dots, f16, haswell, f16, f32, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/8,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, f16, haswell, f16, f32, nk_f16_to_f32, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_f16_,
-                      /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1) // Store as F32
+nk_define_cross_pack_(dots, f16, haswell, f16, f32, nk_b256_vec_t, nk_load_f16x8_to_f32x8_haswell_,
+                      nk_partial_load_f16x8_to_f32x8_haswell_, nk_store_b256_haswell_, nk_partial_store_b32x8_serial_,
+                      /*simd_width=*/8, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_f16_, /*depth_simd_dimensions=*/8,
+                      /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, f16, haswell, f16, f32, nk_b256_vec_t, nk_dot_through_f32_state_haswell_t_,
                            nk_b128_vec_t, nk_dot_through_f32_init_haswell_, nk_load_f16x8_to_f32x8_haswell_,
                            nk_partial_load_f16x8_to_f32x8_haswell_, nk_dot_through_f32_update_haswell_,
@@ -93,29 +96,31 @@ nk_define_cross_packed_(dots, f16, haswell, f16, f32, f32, nk_b256_vec_t, nk_dot
                         /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
 
 /* BF16 GEMM: depth_simd_dimensions=8 (8 bf16s = 16 bytes = 128-bit input) → upcasted to 8×f32 (256-bit) */
-nk_define_cross_pack_size_(dots, bf16, haswell, bf16, f32, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/8,
+/* BF16 GEMM: depth_simd_dimensions=16, raw bf16 storage, unpack(zero, bf16) → f32 inline */
+nk_define_cross_pack_size_(dots, bf16, haswell, bf16, bf16, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/16,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, bf16, haswell, bf16, f32, nk_bf16_to_f32, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_bf16_,
-                      /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1) // Store as F32
-nk_define_cross_symmetric_(dots, bf16, haswell, bf16, f32, nk_b256_vec_t, nk_dot_through_f32_state_haswell_t_,
-                           nk_b128_vec_t, nk_dot_through_f32_init_haswell_, nk_load_bf16x8_to_f32x8_haswell_,
-                           nk_partial_load_bf16x8_to_f32x8_haswell_, nk_dot_through_f32_update_haswell_,
-                           nk_dot_through_f32_finalize_haswell_, nk_store_b128_haswell_,
+nk_define_cross_pack_(dots, bf16, haswell, bf16, bf16, nk_b256_vec_t, nk_load_b256_haswell_,
+                      nk_partial_load_b16x16_serial_, nk_store_b256_haswell_, nk_partial_store_b16x16_serial_,
+                      /*simd_width=*/16, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_bf16_,
+                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, bf16, haswell, bf16, f32, nk_b256_vec_t, nk_dot_bf16x16_state_haswell_t, nk_b128_vec_t,
+                           nk_dot_bf16x16_init_haswell, nk_load_b256_haswell_, nk_partial_load_b16x16_serial_,
+                           nk_dot_bf16x16_update_haswell, nk_dot_bf16x16_finalize_haswell, nk_store_b128_haswell_,
                            nk_partial_store_b32x4_haswell_,
-                           /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
-nk_define_cross_packed_(dots, bf16, haswell, bf16, f32, f32, nk_b256_vec_t, nk_dot_through_f32_state_haswell_t_,
-                        nk_b128_vec_t, nk_dot_through_f32_init_haswell_, nk_load_bf16x8_to_f32x8_haswell_,
-                        nk_partial_load_bf16x8_to_f32x8_haswell_, nk_load_b256_haswell_, nk_partial_load_b32x8_serial_,
-                        nk_dot_through_f32_update_haswell_, nk_dot_through_f32_finalize_haswell_,
-                        nk_store_b128_haswell_, nk_partial_store_b32x4_haswell_,
-                        /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
+                           /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, bf16, haswell, bf16, bf16, f32, nk_b256_vec_t, nk_dot_bf16x16_state_haswell_t,
+                        nk_b128_vec_t, nk_dot_bf16x16_init_haswell, nk_load_b256_haswell_,
+                        nk_partial_load_b16x16_serial_, nk_load_b256_haswell_, nk_partial_load_b16x16_serial_,
+                        nk_dot_bf16x16_update_haswell, nk_dot_bf16x16_finalize_haswell, nk_store_b128_haswell_,
+                        nk_partial_store_b32x4_haswell_,
+                        /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
 
 /* E4M3 GEMM: depth_simd_dimensions=8 (8 e4m3s = 8 bytes) → upcasted to 8×f32 (256-bit) */
 nk_define_cross_pack_size_(dots, e4m3, haswell, e4m3, f32, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/8,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, e4m3, haswell, e4m3, f32, nk_e4m3_to_f32, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_e4m3_,
+nk_define_cross_pack_(dots, e4m3, haswell, e4m3, f32, nk_b256_vec_t, nk_load_e4m3x8_to_f32x8_haswell_,
+                      nk_partial_load_e4m3x8_to_f32x8_haswell_, nk_store_b256_haswell_, nk_partial_store_b32x8_serial_,
+                      /*simd_width=*/8, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e4m3_,
                       /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, e4m3, haswell, e4m3, f32, nk_b256_vec_t, nk_dot_through_f32_state_haswell_t_,
                            nk_b128_vec_t, nk_dot_through_f32_init_haswell_, nk_load_e4m3x8_to_f32x8_haswell_,
@@ -133,8 +138,9 @@ nk_define_cross_packed_(dots, e4m3, haswell, e4m3, f32, f32, nk_b256_vec_t, nk_d
 /* E5M2 GEMM: depth_simd_dimensions=8 (8 e5m2s = 8 bytes) → upcasted to 8×f32 (256-bit) */
 nk_define_cross_pack_size_(dots, e5m2, haswell, e5m2, f32, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/8,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, e5m2, haswell, e5m2, f32, nk_e5m2_to_f32, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_e5m2_,
+nk_define_cross_pack_(dots, e5m2, haswell, e5m2, f32, nk_b256_vec_t, nk_load_e5m2x8_to_f32x8_haswell_,
+                      nk_partial_load_e5m2x8_to_f32x8_haswell_, nk_store_b256_haswell_, nk_partial_store_b32x8_serial_,
+                      /*simd_width=*/8, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e5m2_,
                       /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, e5m2, haswell, e5m2, f32, nk_b256_vec_t, nk_dot_through_f32_state_haswell_t_,
                            nk_b128_vec_t, nk_dot_through_f32_init_haswell_, nk_load_e5m2x8_to_f32x8_haswell_,
@@ -152,8 +158,9 @@ nk_define_cross_packed_(dots, e5m2, haswell, e5m2, f32, f32, nk_b256_vec_t, nk_d
 /* E2M3 GEMM: integer LUT path, depth_simd_dimensions=32 (32 e2m3s = 32 bytes = AVX2 register width) */
 nk_define_cross_pack_size_(dots, e2m3, haswell, e2m3, e2m3, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/32,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, e2m3, haswell, e2m3, e2m3, nk_assign_from_to_, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_e2m3_,
+nk_define_cross_pack_(dots, e2m3, haswell, e2m3, e2m3, nk_b256_vec_t, nk_load_b256_haswell_,
+                      nk_partial_load_b8x32_serial_, nk_store_b256_haswell_, nk_partial_store_b8x32_serial_,
+                      /*simd_width=*/32, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e2m3_,
                       /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, e2m3, haswell, e2m3, f32, nk_b256_vec_t, nk_dot_e2m3x32_state_haswell_t, nk_b128_vec_t,
                            nk_dot_e2m3x32_init_haswell, nk_load_b256_haswell_, nk_partial_load_b8x32_serial_,
@@ -170,8 +177,9 @@ nk_define_cross_packed_(dots, e2m3, haswell, e2m3, e2m3, f32, nk_b256_vec_t, nk_
 /* E3M2 GEMM: integer LUT path, depth_simd_dimensions=32 (32 e3m2s = 32 bytes = AVX2 register width) */
 nk_define_cross_pack_size_(dots, e3m2, haswell, e3m2, e3m2, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/32,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, e3m2, haswell, e3m2, e3m2, nk_assign_from_to_, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_e3m2_,
+nk_define_cross_pack_(dots, e3m2, haswell, e3m2, e3m2, nk_b256_vec_t, nk_load_b256_haswell_,
+                      nk_partial_load_b8x32_serial_, nk_store_b256_haswell_, nk_partial_store_b8x32_serial_,
+                      /*simd_width=*/32, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e3m2_,
                       /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, e3m2, haswell, e3m2, f32, nk_b256_vec_t, nk_dot_e3m2x32_state_haswell_t, nk_b128_vec_t,
                            nk_dot_e3m2x32_init_haswell, nk_load_b256_haswell_, nk_partial_load_b8x32_serial_,
@@ -188,8 +196,10 @@ nk_define_cross_packed_(dots, e3m2, haswell, e3m2, e3m2, f32, nk_b256_vec_t, nk_
 /* I8 GEMM: depth_simd_dimensions=16 (16 i8s = 16 bytes = 128-bit input) */
 nk_define_cross_pack_size_(dots, i8, haswell, i8, i8, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/16,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, i8, haswell, i8, i8, nk_assign_from_to_, /*norm_value_type=*/u32, nk_dots_reduce_sumsq_i8_,
-                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, i8, haswell, i8, i8, nk_b128_vec_t, nk_load_b128_haswell_, nk_partial_load_b8x16_serial_,
+                      nk_store_b128_haswell_, nk_partial_store_b8x16_serial_, /*simd_width=*/16,
+                      /*norm_value_type=*/u32, nk_dots_reduce_sumsq_i8_, /*depth_simd_dimensions=*/16,
+                      /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, i8, haswell, i8, i32, nk_b128_vec_t, nk_dot_i8x16_state_haswell_t, nk_b128_vec_t,
                            nk_dot_i8x16_init_haswell, nk_load_b128_haswell_, nk_partial_load_b8x16_serial_,
                            nk_dot_i8x16_update_haswell, nk_dot_i8x16_finalize_haswell, nk_store_b128_haswell_,
@@ -204,8 +214,10 @@ nk_define_cross_packed_(dots, i8, haswell, i8, i8, i32, nk_b128_vec_t, nk_dot_i8
 /* U8 GEMM: depth_simd_dimensions=16 (16 u8s = 16 bytes = 128-bit input) */
 nk_define_cross_pack_size_(dots, u8, haswell, u8, u8, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/16,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, u8, haswell, u8, u8, nk_assign_from_to_, /*norm_value_type=*/u32, nk_dots_reduce_sumsq_u8_,
-                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, u8, haswell, u8, u8, nk_b128_vec_t, nk_load_b128_haswell_, nk_partial_load_b8x16_serial_,
+                      nk_store_b128_haswell_, nk_partial_store_b8x16_serial_, /*simd_width=*/16,
+                      /*norm_value_type=*/u32, nk_dots_reduce_sumsq_u8_, /*depth_simd_dimensions=*/16,
+                      /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, u8, haswell, u8, u32, nk_b128_vec_t, nk_dot_u8x16_state_haswell_t, nk_b128_vec_t,
                            nk_dot_u8x16_init_haswell, nk_load_b128_haswell_, nk_partial_load_b8x16_serial_,
                            nk_dot_u8x16_update_haswell, nk_dot_u8x16_finalize_haswell, nk_store_b128_haswell_,
@@ -222,9 +234,10 @@ nk_define_cross_packed_(dots, u8, haswell, u8, u8, u32, nk_b128_vec_t, nk_dot_u8
 nk_define_cross_compensated_pack_size_(dots, i4, haswell, i4x2, i4x2,
                                        /*sum_value_type=*/i32, /*norm_value_type=*/u32,
                                        /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/2)
-nk_define_cross_compensated_pack_(dots, i4, haswell, i4x2, i4x2, nk_assign_from_to_,
-                                  /*sum_value_type=*/i32, /*norm_value_type=*/u32, nk_dots_reduce_moments_i4_,
-                                  /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/2)
+nk_define_cross_compensated_pack_(dots, i4, haswell, i4x2, i4x2, nk_b128_vec_t, nk_load_b128_haswell_,
+                                  nk_partial_load_b8x16_serial_, nk_store_b128_haswell_, nk_partial_store_b8x16_serial_,
+                                  /*simd_width=*/16, /*sum_value_type=*/i32, /*norm_value_type=*/u32,
+                                  nk_dots_reduce_moments_i4_, /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/2)
 nk_define_cross_compensated_symmetric_(dots, i4, haswell, i4x2, i32,
                                        /*sum_value_type=*/i32, /*norm_value_type=*/u32, nk_b128_vec_t,
                                        nk_dot_i4x32_state_haswell_t, nk_b128_vec_t, nk_dot_i4x32_init_haswell,
@@ -249,8 +262,9 @@ nk_define_cross_compensated_packed_(dots, i4, haswell, i4x2, i4x2, i32,
  * Note: dimensions_per_value=2 because 2 nibbles (u4 values) are packed per byte */
 nk_define_cross_pack_size_(dots, u4, haswell, u4x2, u4x2, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/32,
                            /*dimensions_per_value=*/2)
-nk_define_cross_pack_(dots, u4, haswell, u4x2, u4x2, nk_assign_from_to_, /*norm_value_type=*/u32,
-                      nk_dots_reduce_sumsq_u4_,
+nk_define_cross_pack_(dots, u4, haswell, u4x2, u4x2, nk_b128_vec_t, nk_load_b128_haswell_,
+                      nk_partial_load_b8x16_serial_, nk_store_b128_haswell_, nk_partial_store_b8x16_serial_,
+                      /*simd_width=*/16, /*norm_value_type=*/u32, nk_dots_reduce_sumsq_u4_,
                       /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/2)
 nk_define_cross_symmetric_(dots, u4, haswell, u4x2, u32, nk_b128_vec_t, nk_dot_u4x32_state_haswell_t, nk_b128_vec_t,
                            nk_dot_u4x32_init_haswell, nk_load_b128_haswell_, nk_partial_load_b4x32_serial_,
@@ -266,8 +280,9 @@ nk_define_cross_packed_(dots, u4, haswell, u4x2, u4x2, u32, nk_b128_vec_t, nk_do
 /* U1 GEMM: depth_simd_dimensions=128 (128 bits = 16 bytes) */
 nk_define_cross_pack_size_(dots, u1, haswell, u1x8, u1x8, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/128,
                            /*dimensions_per_value=*/8)
-nk_define_cross_pack_(dots, u1, haswell, u1x8, u1x8, nk_assign_from_to_, /*norm_value_type=*/u32,
-                      nk_dots_reduce_sum_u1_,
+nk_define_cross_pack_(dots, u1, haswell, u1x8, u1x8, nk_b128_vec_t, nk_load_b128_haswell_,
+                      nk_partial_load_b8x16_serial_, nk_store_b128_haswell_, nk_partial_store_b8x16_serial_,
+                      /*simd_width=*/16, /*norm_value_type=*/u32, nk_dots_reduce_sum_u1_,
                       /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/8)
 nk_define_cross_symmetric_(dots, u1, haswell, u1x8, u32, nk_b128_vec_t, nk_dot_u1x128_state_haswell_t, nk_b128_vec_t,
                            nk_dot_u1x128_init_haswell, nk_load_b128_haswell_, nk_partial_load_b1x128_serial_,

package/include/numkong/dots/icelake.h

@@ -8,11 +8,11 @@
  *
  *  @section ice_dots_instructions Relevant Instructions
  *
- *      Intrinsic                   Instruction                     Ice         Genoa
- *      _mm512_dpbusd_epi32         VPDPBUSD (ZMM, ZMM, ZMM)        5cy @ p0    4cy @ p01
- *      _mm512_dpwssd_epi32         VPDPWSSD (ZMM, ZMM, ZMM)        5cy @ p0    4cy @ p01
- *      _mm512_cvtepi8_epi32        VPMOVSXBD (ZMM, XMM)            3cy @ p5    3cy @ p12
- *      _mm512_loadu_si512          VMOVDQU64 (ZMM, M512)           7cy @ p23   7cy @ p23
+ *      Intrinsic             Instruction               Icelake    Genoa
+ *      _mm512_dpbusd_epi32   VPDPBUSD (ZMM, ZMM, ZMM)  5cy @ p0   4cy @ p01
+ *      _mm512_dpwssd_epi32   VPDPWSSD (ZMM, ZMM, ZMM)  5cy @ p0   4cy @ p01
+ *      _mm512_cvtepi8_epi32  VPMOVSXBD (ZMM, XMM)      3cy @ p5   3cy @ p12
+ *      _mm512_loadu_si512    VMOVDQU64 (ZMM, M512)     7cy @ p23  7cy @ p23
  *
  *  Ice Lake's VNNI instructions accelerate int8 GEMM by computing 4-element dot products per lane.
  *  VPDPBUSD/VPDPWSSD bottleneck on port 0, limiting throughput to 1/cy. AMD Genoa achieves 0.5/cy
@@ -45,9 +45,11 @@ extern "C" {
 nk_define_cross_compensated_pack_size_(dots, i8, icelake, i8, i8,
                                        /*sum_value_type=*/i32, /*norm_value_type=*/u32,
                                        /*depth_simd_dimensions=*/64, /*dimensions_per_value=*/1)
-nk_define_cross_compensated_pack_(dots, i8, icelake, i8, i8, nk_assign_from_to_,
-                                  /*sum_value_type=*/i32, /*norm_value_type=*/u32, nk_dots_reduce_moments_i8_,
-                                  /*depth_simd_dimensions=*/64, /*dimensions_per_value=*/1)
+nk_define_cross_compensated_pack_(dots, i8, icelake, i8, i8, nk_b512_vec_t, nk_load_b512_skylake_,
+                                  nk_partial_load_b8x64_skylake_, nk_store_b512_skylake_,
+                                  nk_partial_store_b8x64_skylake_, /*simd_width=*/64, /*sum_value_type=*/i32,
+                                  /*norm_value_type=*/u32, nk_dots_reduce_moments_i8_, /*depth_simd_dimensions=*/64,
+                                  /*dimensions_per_value=*/1)
 nk_define_cross_compensated_symmetric_(dots, i8, icelake, i8, i32,
                                        /*sum_value_type=*/i32, /*norm_value_type=*/u32, nk_b512_vec_t,
                                        nk_dot_i8x64_state_icelake_t, nk_b128_vec_t, nk_dot_i8x64_init_icelake,
@@ -72,9 +74,11 @@ nk_define_cross_compensated_packed_(dots, i8, icelake, i8, i8, i32,
 nk_define_cross_compensated_pack_size_(dots, u8, icelake, u8, u8,
                                        /*sum_value_type=*/u32, /*norm_value_type=*/u32,
                                        /*depth_simd_dimensions=*/64, /*dimensions_per_value=*/1)
-nk_define_cross_compensated_pack_(dots, u8, icelake, u8, u8, nk_assign_from_to_,
-                                  /*sum_value_type=*/u32, /*norm_value_type=*/u32, nk_dots_reduce_moments_u8_,
-                                  /*depth_simd_dimensions=*/64, /*dimensions_per_value=*/1)
+nk_define_cross_compensated_pack_(dots, u8, icelake, u8, u8, nk_b512_vec_t, nk_load_b512_skylake_,
+                                  nk_partial_load_b8x64_skylake_, nk_store_b512_skylake_,
+                                  nk_partial_store_b8x64_skylake_, /*simd_width=*/64, /*sum_value_type=*/u32,
+                                  /*norm_value_type=*/u32, nk_dots_reduce_moments_u8_, /*depth_simd_dimensions=*/64,
+                                  /*dimensions_per_value=*/1)
 nk_define_cross_compensated_symmetric_(dots, u8, icelake, u8, u32,
                                        /*sum_value_type=*/u32, /*norm_value_type=*/u32, nk_b512_vec_t,
                                        nk_dot_u8x64_state_icelake_t, nk_b128_vec_t, nk_dot_u8x64_init_icelake,
@@ -99,9 +103,11 @@ nk_define_cross_compensated_packed_(dots, u8, icelake, u8, u8, u32,
 nk_define_cross_compensated_pack_size_(dots, i4, icelake, i4x2, i4x2,
                                        /*sum_value_type=*/i32, /*norm_value_type=*/u32,
                                        /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/2)
-nk_define_cross_compensated_pack_(dots, i4, icelake, i4x2, i4x2, nk_assign_from_to_,
-                                  /*sum_value_type=*/i32, /*norm_value_type=*/u32, nk_dots_reduce_moments_i4_,
-                                  /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/2)
+nk_define_cross_compensated_pack_(dots, i4, icelake, i4x2, i4x2, nk_b512_vec_t, nk_load_b512_skylake_,
+                                  nk_partial_load_b8x64_skylake_, nk_store_b512_skylake_,
+                                  nk_partial_store_b8x64_skylake_, /*simd_width=*/64, /*sum_value_type=*/i32,
+                                  /*norm_value_type=*/u32, nk_dots_reduce_moments_i4_, /*depth_simd_dimensions=*/64,
+                                  /*dimensions_per_value=*/2)
 nk_define_cross_compensated_symmetric_(dots, i4, icelake, i4x2, i32,
                                        /*sum_value_type=*/i32, /*norm_value_type=*/u32, nk_b512_vec_t,
                                        nk_dot_i4x128_state_icelake_t, nk_b128_vec_t, nk_dot_i4x128_init_icelake,
@@ -125,8 +131,10 @@ nk_define_cross_compensated_packed_(dots, i4, icelake, i4x2, i4x2, i32,
 /* U4 GEMM: depth_simd_dimensions=128 (128 nibbles = 64 bytes = full cache line) */
 nk_define_cross_pack_size_(dots, u4, icelake, u4x2, u4x2, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/128,
                            /*dimensions_per_value=*/2)
-nk_define_cross_pack_(dots, u4, icelake, u4x2, u4x2, nk_assign_from_to_, /*norm_value_type=*/u32,
-                      nk_dots_reduce_sumsq_u4_, /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/2)
+nk_define_cross_pack_(dots, u4, icelake, u4x2, u4x2, nk_b512_vec_t, nk_load_b512_skylake_,
+                      nk_partial_load_b8x64_skylake_, nk_store_b512_skylake_, nk_partial_store_b8x64_skylake_,
+                      /*simd_width=*/64, /*norm_value_type=*/u32, nk_dots_reduce_sumsq_u4_,
+                      /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/2)
 
 nk_define_cross_symmetric_(dots, u4, icelake, u4x2, u32, nk_b512_vec_t, nk_dot_u4x128_state_icelake_t, nk_b128_vec_t,
                            nk_dot_u4x128_init_icelake, nk_load_b512_skylake_, nk_partial_load_b4x128_skylake_,
@@ -142,8 +150,9 @@ nk_define_cross_packed_(dots, u4, icelake, u4x2, u4x2, u32, nk_b512_vec_t, nk_do
 /* U1 GEMM: depth_simd_dimensions=512 (512 bits = 64 bytes = full cache line) */
 nk_define_cross_pack_size_(dots, u1, icelake, u1x8, u1x8, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/512,
                            /*dimensions_per_value=*/8)
-nk_define_cross_pack_(dots, u1, icelake, u1x8, u1x8, nk_assign_from_to_, /*norm_value_type=*/u32,
-                      nk_dots_reduce_sum_u1_,
+nk_define_cross_pack_(dots, u1, icelake, u1x8, u1x8, nk_b512_vec_t, nk_load_b512_skylake_,
+                      nk_partial_load_b8x64_skylake_, nk_store_b512_skylake_, nk_partial_store_b8x64_skylake_,
+                      /*simd_width=*/64, /*norm_value_type=*/u32, nk_dots_reduce_sum_u1_,
                       /*depth_simd_dimensions=*/512, /*dimensions_per_value=*/8)
 nk_define_cross_symmetric_(dots, u1, icelake, u1x8, u32, nk_b512_vec_t, nk_dot_u1x512_state_icelake_t, nk_b128_vec_t,
                            nk_dot_u1x512_init_icelake, nk_load_b512_skylake_, nk_partial_load_b1x512_skylake_,