numkong 7.0.0 → 7.4.2

package/include/numkong/dots/sierra.h

@@ -34,8 +34,9 @@ extern "C" {
 /* I8 GEMM: depth_simd_dimensions=32 (32 i8s = 32 bytes = AVX2 register width) */
 nk_define_cross_pack_size_(dots, i8, sierra, i8, i8, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/32,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, i8, sierra, i8, i8, nk_assign_from_to_, /*norm_value_type=*/u32, nk_dots_reduce_sumsq_i8_,
-                      /*depth_simd_dimensions=*/32,
+nk_define_cross_pack_(dots, i8, sierra, i8, i8, nk_b128_vec_t, nk_load_b128_haswell_, nk_partial_load_b8x16_serial_,
+                      nk_store_b128_haswell_, nk_partial_store_b8x16_serial_, /*simd_width=*/16,
+                      /*norm_value_type=*/u32, nk_dots_reduce_sumsq_i8_, /*depth_simd_dimensions=*/16,
                       /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, i8, sierra, i8, i32, nk_b256_vec_t, nk_dot_i8x32_state_sierra_t, nk_b128_vec_t,
                            nk_dot_i8x32_init_sierra, nk_load_b256_haswell_, nk_partial_load_b8x32_serial_,
@@ -51,8 +52,9 @@ nk_define_cross_packed_(dots, i8, sierra, i8, i8, i32, nk_b256_vec_t, nk_dot_i8x
 /* U8 GEMM: depth_simd_dimensions=32 (32 u8s = 32 bytes = AVX2 register width) */
 nk_define_cross_pack_size_(dots, u8, sierra, u8, u8, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/32,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, u8, sierra, u8, u8, nk_assign_from_to_, /*norm_value_type=*/u32, nk_dots_reduce_sumsq_u8_,
-                      /*depth_simd_dimensions=*/32,
+nk_define_cross_pack_(dots, u8, sierra, u8, u8, nk_b128_vec_t, nk_load_b128_haswell_, nk_partial_load_b8x16_serial_,
+                      nk_store_b128_haswell_, nk_partial_store_b8x16_serial_, /*simd_width=*/16,
+                      /*norm_value_type=*/u32, nk_dots_reduce_sumsq_u8_, /*depth_simd_dimensions=*/16,
                       /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, u8, sierra, u8, u32, nk_b256_vec_t, nk_dot_u8x32_state_sierra_t, nk_b128_vec_t,
                            nk_dot_u8x32_init_sierra, nk_load_b256_haswell_, nk_partial_load_b8x32_serial_,
@@ -68,9 +70,10 @@ nk_define_cross_packed_(dots, u8, sierra, u8, u8, u32, nk_b256_vec_t, nk_dot_u8x
 /* E2M3 GEMM via DPBUSD integer path: depth_simd_dimensions=32 (32 e2m3s = 32 bytes = AVX2 register width) */
 nk_define_cross_pack_size_(dots, e2m3, sierra, e2m3, e2m3, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/32,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, e2m3, sierra, e2m3, e2m3, nk_assign_from_to_, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_e2m3_, /*depth_simd_dimensions=*/32,
-                      /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, e2m3, sierra, e2m3, e2m3, nk_b128_vec_t, nk_load_b128_haswell_,
+                      nk_partial_load_b8x16_serial_, nk_store_b128_haswell_, nk_partial_store_b8x16_serial_,
+                      /*simd_width=*/16, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e2m3_,
+                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, e2m3, sierra, e2m3, f32, nk_b256_vec_t, nk_dot_e2m3x32_state_sierra_t, nk_b128_vec_t,
                            nk_dot_e2m3x32_init_sierra, nk_load_b256_haswell_, nk_partial_load_b8x32_serial_,
                            nk_dot_e2m3x32_update_sierra, nk_dot_e2m3x32_finalize_sierra, nk_store_b128_haswell_,

package/include/numkong/dots/skylake.h

@@ -8,11 +8,11 @@
  *
  *  @section skylake_dots_instructions Relevant Instructions
  *
- *      Intrinsic                   Instruction                     SKL         ICL         Genoa
- *      _mm512_fmadd_ps             VFMADD132PS (ZMM, ZMM, ZMM)     4cy @ p05   4cy @ p05   4cy @ p01
- *      _mm512_fmadd_pd             VFMADD132PD (ZMM, ZMM, ZMM)     4cy @ p05   4cy @ p05   4cy @ p01
- *      _mm512_cvtph_ps             VCVTPH2PS (ZMM, YMM)            5cy @ p05   5cy @ p05   5cy @ p01
- *      _mm512_loadu_ps             VMOVUPS (ZMM, M512)             7cy @ p23   7cy @ p23   7cy @ p23
+ *      Intrinsic        Instruction                  SKL        ICL        Genoa
+ *      _mm512_fmadd_ps  VFMADD132PS (ZMM, ZMM, ZMM)  4cy @ p05  4cy @ p05  4cy @ p01
+ *      _mm512_fmadd_pd  VFMADD132PD (ZMM, ZMM, ZMM)  4cy @ p05  4cy @ p05  4cy @ p01
+ *      _mm512_cvtph_ps  VCVTPH2PS (ZMM, YMM)         5cy @ p05  5cy @ p05  5cy @ p01
+ *      _mm512_loadu_ps  VMOVUPS (ZMM, M512)          7cy @ p23  7cy @ p23  7cy @ p23
  *
  *  GEMM micro-kernels tile the K dimension to maximize FMA throughput. Skylake-X server chips with
  *  dual FMA units achieve 0.5cy throughput, enabling 32 FLOPs/cycle for f32 or 16 FLOPs/cycle for f64.
@@ -41,8 +41,9 @@ extern "C" {
 /* F64 GEMM: depth_simd_dimensions=8 (8 f64s = 64 bytes = 1 cache line) */
 nk_define_cross_pack_size_(dots, f64, skylake, f64, f64, /*norm_value_type=*/f64, /*depth_simd_dimensions=*/8,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, f64, skylake, f64, f64, nk_assign_from_to_, /*norm_value_type=*/f64,
-                      nk_dots_reduce_sumsq_f64_,
+nk_define_cross_pack_(dots, f64, skylake, f64, f64, nk_b512_vec_t, nk_load_b512_skylake_,
+                      nk_partial_load_b64x8_skylake_, nk_store_b512_skylake_, nk_partial_store_b64x8_skylake_,
+                      /*simd_width=*/8, /*norm_value_type=*/f64, nk_dots_reduce_sumsq_f64_,
                       /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, f64, skylake, f64, f64, nk_b512_vec_t, nk_dot_f64x8_state_skylake_t, nk_b256_vec_t,
                            nk_dot_f64x8_init_skylake, nk_load_b512_skylake_, nk_partial_load_b64x8_skylake_,
@@ -58,8 +59,9 @@ nk_define_cross_packed_(dots, f64, skylake, f64, f64, f64, nk_b512_vec_t, nk_dot
 /* F32 GEMM: depth_simd_dimensions=8 (8 f32s = 32 bytes = half cache line) */
 nk_define_cross_pack_size_(dots, f32, skylake, f32, f32, /*norm_value_type=*/f64, /*depth_simd_dimensions=*/8,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, f32, skylake, f32, f32, nk_assign_from_to_, /*norm_value_type=*/f64,
-                      nk_dots_reduce_sumsq_f32_,
+nk_define_cross_pack_(dots, f32, skylake, f32, f32, nk_b512_vec_t, nk_load_b512_skylake_,
+                      nk_partial_load_b32x16_skylake_, nk_store_b512_skylake_, nk_partial_store_b32x16_skylake_,
+                      /*simd_width=*/16, /*norm_value_type=*/f64, nk_dots_reduce_sumsq_f32_,
                       /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, f32, skylake, f32, f64, nk_b256_vec_t, nk_dot_f32x8_state_skylake_t, nk_b256_vec_t,
                            nk_dot_f32x8_init_skylake, nk_load_b256_haswell_, nk_partial_load_b32x8_skylake_,
@@ -73,29 +75,32 @@ nk_define_cross_packed_(dots, f32, skylake, f32, f32, f64, nk_b256_vec_t, nk_dot
                         /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
 
 /* BF16 GEMM: depth_simd_dimensions=16 (16 bf16s = 32 bytes = half cache line), F32 accumulator */
-nk_define_cross_pack_size_(dots, bf16, skylake, bf16, f32, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/16,
+/* BF16 GEMM: depth_simd_dimensions=32, raw bf16 storage, unpack(zero, bf16) → f32 inline */
+nk_define_cross_pack_size_(dots, bf16, skylake, bf16, bf16, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/32,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, bf16, skylake, bf16, f32, nk_bf16_to_f32, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_bf16_,
-                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
-nk_define_cross_symmetric_(dots, bf16, skylake, bf16, f32, nk_b512_vec_t, nk_dot_through_f32_state_skylake_t_,
-                           nk_b128_vec_t, nk_dot_through_f32_init_skylake_, nk_load_bf16x16_to_f32x16_skylake_,
-                           nk_partial_load_bf16x16_to_f32x16_skylake_, nk_dot_through_f32_update_skylake_,
-                           nk_dot_through_f32_finalize_skylake_, nk_store_b128_haswell_,
+nk_define_cross_pack_(dots, bf16, skylake, bf16, bf16, nk_b512_vec_t, nk_load_b512_skylake_,
+                      nk_partial_load_b16x32_skylake_, nk_store_b512_skylake_, nk_partial_store_b16x32_skylake_,
+                      /*simd_width=*/32, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_bf16_,
+                      /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, bf16, skylake, bf16, f32, nk_b512_vec_t, nk_dot_bf16x32_state_skylake_t, nk_b128_vec_t,
+                           nk_dot_bf16x32_init_skylake, nk_load_b512_skylake_, nk_partial_load_b16x32_skylake_,
+                           nk_dot_bf16x32_update_skylake, nk_dot_bf16x32_finalize_skylake, nk_store_b128_haswell_,
                            nk_partial_store_b32x4_skylake_,
-                           /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
-nk_define_cross_packed_(dots, bf16, skylake, bf16, f32, f32, nk_b512_vec_t, nk_dot_through_f32_state_skylake_t_,
-                        nk_b128_vec_t, nk_dot_through_f32_init_skylake_, nk_load_bf16x16_to_f32x16_skylake_,
-                        nk_partial_load_bf16x16_to_f32x16_skylake_, nk_load_b512_skylake_,
-                        nk_partial_load_b32x16_skylake_, nk_dot_through_f32_update_skylake_,
-                        nk_dot_through_f32_finalize_skylake_, nk_store_b128_haswell_, nk_partial_store_b32x4_skylake_,
-                        /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+                           /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, bf16, skylake, bf16, bf16, f32, nk_b512_vec_t, nk_dot_bf16x32_state_skylake_t,
+                        nk_b128_vec_t, nk_dot_bf16x32_init_skylake, nk_load_b512_skylake_,
+                        nk_partial_load_b16x32_skylake_, nk_load_b512_skylake_, nk_partial_load_b16x32_skylake_,
+                        nk_dot_bf16x32_update_skylake, nk_dot_bf16x32_finalize_skylake, nk_store_b128_haswell_,
+                        nk_partial_store_b32x4_skylake_,
+                        /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
 
 /* F16 GEMM: depth_simd_dimensions=16 (16 f16s = 32 bytes = half cache line), F32 accumulator */
 nk_define_cross_pack_size_(dots, f16, skylake, f16, f32, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/16,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, f16, skylake, f16, f32, nk_f16_to_f32, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_f16_,
-                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, f16, skylake, f16, f32, nk_b512_vec_t, nk_load_f16x16_to_f32x16_skylake_,
+                      nk_partial_load_f16x16_to_f32x16_skylake_, nk_store_b512_skylake_,
+                      nk_partial_store_b32x16_skylake_, /*simd_width=*/16, /*norm_value_type=*/f32,
+                      nk_dots_reduce_sumsq_f16_, /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, f16, skylake, f16, f32, nk_b512_vec_t, nk_dot_through_f32_state_skylake_t_,
                            nk_b128_vec_t, nk_dot_through_f32_init_skylake_, nk_load_f16x16_to_f32x16_skylake_,
                            nk_partial_load_f16x16_to_f32x16_skylake_, nk_dot_through_f32_update_skylake_,
@@ -112,9 +117,10 @@ nk_define_cross_packed_(dots, f16, skylake, f16, f32, f32, nk_b512_vec_t, nk_dot
 /* E4M3 GEMM: depth_simd_dimensions=16 (16 e4m3s = 16 bytes = quarter cache line), F32 accumulator */
 nk_define_cross_pack_size_(dots, e4m3, skylake, e4m3, f32, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/16,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, e4m3, skylake, e4m3, f32, nk_e4m3_to_f32, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_e4m3_,
-                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, e4m3, skylake, e4m3, f32, nk_b512_vec_t, nk_load_e4m3x16_to_f32x16_skylake_,
+                      nk_partial_load_e4m3x16_to_f32x16_skylake_, nk_store_b512_skylake_,
+                      nk_partial_store_b32x16_skylake_, /*simd_width=*/16, /*norm_value_type=*/f32,
+                      nk_dots_reduce_sumsq_e4m3_, /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, e4m3, skylake, e4m3, f32, nk_b512_vec_t, nk_dot_through_f32_state_skylake_t_,
                            nk_b128_vec_t, nk_dot_through_f32_init_skylake_, nk_load_e4m3x16_to_f32x16_skylake_,
                            nk_partial_load_e4m3x16_to_f32x16_skylake_, nk_dot_through_f32_update_skylake_,
@@ -131,9 +137,10 @@ nk_define_cross_packed_(dots, e4m3, skylake, e4m3, f32, f32, nk_b512_vec_t, nk_d
 /* E5M2 GEMM: depth_simd_dimensions=16 (16 e5m2s = 16 bytes = quarter cache line), F32 accumulator */
 nk_define_cross_pack_size_(dots, e5m2, skylake, e5m2, f32, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/16,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, e5m2, skylake, e5m2, f32, nk_e5m2_to_f32, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_e5m2_,
-                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, e5m2, skylake, e5m2, f32, nk_b512_vec_t, nk_load_e5m2x16_to_f32x16_skylake_,
+                      nk_partial_load_e5m2x16_to_f32x16_skylake_, nk_store_b512_skylake_,
+                      nk_partial_store_b32x16_skylake_, /*simd_width=*/16, /*norm_value_type=*/f32,
+                      nk_dots_reduce_sumsq_e5m2_, /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, e5m2, skylake, e5m2, f32, nk_b512_vec_t, nk_dot_through_f32_state_skylake_t_,
                            nk_b128_vec_t, nk_dot_through_f32_init_skylake_, nk_load_e5m2x16_to_f32x16_skylake_,
                            nk_partial_load_e5m2x16_to_f32x16_skylake_, nk_dot_through_f32_update_skylake_,
@@ -150,8 +157,10 @@ nk_define_cross_packed_(dots, e5m2, skylake, e5m2, f32, f32, nk_b512_vec_t, nk_d
 /* E2M3 GEMM: integer LUT path, depth_simd_dimensions=64 (64 e2m3s = 64 bytes = AVX-512 register width) */
 nk_define_cross_pack_size_(dots, e2m3, skylake, e2m3, e2m3, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/64,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, e2m3, skylake, e2m3, e2m3, nk_assign_from_to_, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_e2m3_, /*depth_simd_dimensions=*/64, /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, e2m3, skylake, e2m3, e2m3, nk_b512_vec_t, nk_load_b512_skylake_,
+                      nk_partial_load_b8x64_skylake_, nk_store_b512_skylake_, nk_partial_store_b8x64_skylake_,
+                      /*simd_width=*/64, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e2m3_,
+                      /*depth_simd_dimensions=*/64, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, e2m3, skylake, e2m3, f32, nk_b512_vec_t, nk_dot_e2m3x64_state_skylake_t, nk_b128_vec_t,
                            nk_dot_e2m3x64_init_skylake, nk_load_b512_skylake_, nk_partial_load_b8x64_skylake_,
                            nk_dot_e2m3x64_update_skylake, nk_dot_e2m3x64_finalize_skylake, nk_store_b128_haswell_,
@@ -167,8 +176,10 @@ nk_define_cross_packed_(dots, e2m3, skylake, e2m3, e2m3, f32, nk_b512_vec_t, nk_
 /* E3M2 GEMM: integer LUT path, depth_simd_dimensions=64 (64 e3m2s = 64 bytes = AVX-512 register width) */
 nk_define_cross_pack_size_(dots, e3m2, skylake, e3m2, e3m2, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/64,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, e3m2, skylake, e3m2, e3m2, nk_assign_from_to_, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_e3m2_, /*depth_simd_dimensions=*/64, /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, e3m2, skylake, e3m2, e3m2, nk_b512_vec_t, nk_load_b512_skylake_,
+                      nk_partial_load_b8x64_skylake_, nk_store_b512_skylake_, nk_partial_store_b8x64_skylake_,
+                      /*simd_width=*/64, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e3m2_,
+                      /*depth_simd_dimensions=*/64, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, e3m2, skylake, e3m2, f32, nk_b512_vec_t, nk_dot_e3m2x64_state_skylake_t, nk_b128_vec_t,
                            nk_dot_e3m2x64_init_skylake, nk_load_b512_skylake_, nk_partial_load_b8x64_skylake_,
                            nk_dot_e3m2x64_update_skylake, nk_dot_e3m2x64_finalize_skylake, nk_store_b128_haswell_,