numkong 7.0.0 → 7.4.1

package/include/numkong/dots/loongsonasx.h

@@ -0,0 +1,176 @@
+/**
+ *  @brief SIMD-accelerated Batched Dot Products for LoongArch ASX.
+ *  @file include/numkong/dots/loongsonasx.h
+ *  @author Ash Vardanian
+ *  @date March 23, 2026
+ *
+ *  @sa include/numkong/dots.h
+ *
+ *  GEMM kernels use tiled dot products with 4-way parallel accumulation to hide FMA latency.
+ *  LASX is 256-bit, matching AVX2 in register width.
+ *  Type-specific tile sizes: f32/f64 use depth_simd_dimensions=4, i8/u8 use depth_simd_dimensions=16.
+ */
+#ifndef NK_DOTS_LOONGSONASX_H
+#define NK_DOTS_LOONGSONASX_H
+
+#if NK_TARGET_LOONGARCH_
+#if NK_TARGET_LOONGSONASX
+
+#include "numkong/dot/loongsonasx.h"
+#include "numkong/cast/loongsonasx.h"
+#include "numkong/cast.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/* F32 GEMM: depth_simd_dimensions=8 (8 f32s = 256-bit input → f64 accumulation via low/high widening) */
+nk_define_cross_pack_size_(dots, f32, loongsonasx, f32, f32, /*norm_value_type=*/f64, /*depth_simd_dimensions=*/8,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, f32, loongsonasx, f32, f32, nk_b256_vec_t, nk_load_b256_loongsonasx_,
+                      nk_partial_load_b32x8_serial_, nk_store_b256_loongsonasx_, nk_partial_store_b32x8_serial_,
+                      /*simd_width=*/8, /*norm_value_type=*/f64, nk_dots_reduce_sumsq_f32_,
+                      /*depth_simd_dimensions=*/2, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, f32, loongsonasx, f32, f64, nk_b256_vec_t, nk_dot_f32x8_state_loongsonasx_t,
+                           nk_b256_vec_t, nk_dot_f32x8_init_loongsonasx, nk_load_b256_loongsonasx_,
+                           nk_partial_load_b32x8_serial_, nk_dot_f32x8_update_loongsonasx,
+                           nk_dot_f32x8_finalize_loongsonasx, nk_store_b256_loongsonasx_,
+                           nk_partial_store_b64x4_serial_,
+                           /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, f32, loongsonasx, f32, f32, f64, nk_b256_vec_t, nk_dot_f32x8_state_loongsonasx_t,
+                        nk_b256_vec_t, nk_dot_f32x8_init_loongsonasx, nk_load_b256_loongsonasx_,
+                        nk_partial_load_b32x8_serial_, nk_load_b256_loongsonasx_, nk_partial_load_b32x8_serial_,
+                        nk_dot_f32x8_update_loongsonasx, nk_dot_f32x8_finalize_loongsonasx, nk_store_b256_loongsonasx_,
+                        nk_partial_store_b64x4_serial_,
+                        /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
+
+/* F64 GEMM: depth_simd_dimensions=4 (4 f64s = 256-bit = full LASX register) */
+nk_define_cross_pack_size_(dots, f64, loongsonasx, f64, f64, /*norm_value_type=*/f64, /*depth_simd_dimensions=*/4,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, f64, loongsonasx, f64, f64, nk_b256_vec_t, nk_load_b256_loongsonasx_,
+                      nk_partial_load_b64x4_serial_, nk_store_b256_loongsonasx_, nk_partial_store_b64x4_serial_,
+                      /*simd_width=*/4, /*norm_value_type=*/f64, nk_dots_reduce_sumsq_f64_,
+                      /*depth_simd_dimensions=*/2, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, f64, loongsonasx, f64, f64, nk_b256_vec_t, nk_dot_f64x4_state_loongsonasx_t,
+                           nk_b256_vec_t, nk_dot_f64x4_init_loongsonasx, nk_load_b256_loongsonasx_,
+                           nk_partial_load_b64x4_serial_, nk_dot_f64x4_update_loongsonasx,
+                           nk_dot_f64x4_finalize_loongsonasx, nk_store_b256_loongsonasx_,
+                           nk_partial_store_b64x4_serial_,
+                           /*depth_simd_dimensions=*/4, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, f64, loongsonasx, f64, f64, f64, nk_b256_vec_t, nk_dot_f64x4_state_loongsonasx_t,
+                        nk_b256_vec_t, nk_dot_f64x4_init_loongsonasx, nk_load_b256_loongsonasx_,
+                        nk_partial_load_b64x4_serial_, nk_load_b256_loongsonasx_, nk_partial_load_b64x4_serial_,
+                        nk_dot_f64x4_update_loongsonasx, nk_dot_f64x4_finalize_loongsonasx, nk_store_b256_loongsonasx_,
+                        nk_partial_store_b64x4_serial_,
+                        /*depth_simd_dimensions=*/4, /*dimensions_per_value=*/1)
+
+/* I8 GEMM: depth_simd_dimensions=32 (32 i8s = 256-bit input → i32 accumulation) */
+nk_define_cross_pack_size_(dots, i8, loongsonasx, i8, i8, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/32,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, i8, loongsonasx, i8, i8, nk_b256_vec_t, nk_load_b256_loongsonasx_,
+                      nk_partial_load_b8x32_serial_, nk_store_b256_loongsonasx_, nk_partial_store_b8x32_serial_,
+                      /*simd_width=*/32, /*norm_value_type=*/u32, nk_dots_reduce_sumsq_i8_,
+                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, i8, loongsonasx, i8, i32, nk_b256_vec_t, nk_dot_i8x32_state_loongsonasx_t,
+                           nk_b128_vec_t, nk_dot_i8x32_init_loongsonasx, nk_load_b256_loongsonasx_,
+                           nk_partial_load_b8x32_serial_, nk_dot_i8x32_update_loongsonasx,
+                           nk_dot_i8x32_finalize_loongsonasx, nk_store_b128_loongsonasx_,
+                           nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, i8, loongsonasx, i8, i8, i32, nk_b256_vec_t, nk_dot_i8x32_state_loongsonasx_t,
+                        nk_b128_vec_t, nk_dot_i8x32_init_loongsonasx, nk_load_b256_loongsonasx_,
+                        nk_partial_load_b8x32_serial_, nk_load_b256_loongsonasx_, nk_partial_load_b8x32_serial_,
+                        nk_dot_i8x32_update_loongsonasx, nk_dot_i8x32_finalize_loongsonasx, nk_store_b128_loongsonasx_,
+                        nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
+
+/* U8 GEMM: depth_simd_dimensions=32 (32 u8s = 256-bit input → u32 accumulation) */
+nk_define_cross_pack_size_(dots, u8, loongsonasx, u8, u8, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/32,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, u8, loongsonasx, u8, u8, nk_b256_vec_t, nk_load_b256_loongsonasx_,
+                      nk_partial_load_b8x32_serial_, nk_store_b256_loongsonasx_, nk_partial_store_b8x32_serial_,
+                      /*simd_width=*/32, /*norm_value_type=*/u32, nk_dots_reduce_sumsq_u8_,
+                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, u8, loongsonasx, u8, u32, nk_b256_vec_t, nk_dot_u8x32_state_loongsonasx_t,
+                           nk_b128_vec_t, nk_dot_u8x32_init_loongsonasx, nk_load_b256_loongsonasx_,
+                           nk_partial_load_b8x32_serial_, nk_dot_u8x32_update_loongsonasx,
+                           nk_dot_u8x32_finalize_loongsonasx, nk_store_b128_loongsonasx_,
+                           nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, u8, loongsonasx, u8, u8, u32, nk_b256_vec_t, nk_dot_u8x32_state_loongsonasx_t,
+                        nk_b128_vec_t, nk_dot_u8x32_init_loongsonasx, nk_load_b256_loongsonasx_,
+                        nk_partial_load_b8x32_serial_, nk_load_b256_loongsonasx_, nk_partial_load_b8x32_serial_,
+                        nk_dot_u8x32_update_loongsonasx, nk_dot_u8x32_finalize_loongsonasx, nk_store_b128_loongsonasx_,
+                        nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/1)
+
+/* U1 GEMM: depth_simd_dimensions=256 (256 bits = 32 bytes per tile → u32 popcount via XVPCNT.W) */
+nk_define_cross_pack_size_(dots, u1, loongsonasx, u1x8, u1x8, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/256,
+                           /*dimensions_per_value=*/8)
+nk_define_cross_pack_(dots, u1, loongsonasx, u1x8, u1x8, nk_b256_vec_t, nk_load_b256_loongsonasx_,
+                      nk_partial_load_b8x32_serial_, nk_store_b256_loongsonasx_, nk_partial_store_b8x32_serial_,
+                      /*simd_width=*/32, /*norm_value_type=*/u32, nk_dots_reduce_sum_u1_,
+                      /*depth_simd_dimensions=*/256, /*dimensions_per_value=*/8)
+nk_define_cross_symmetric_(dots, u1, loongsonasx, u1x8, u32, nk_b256_vec_t, nk_dot_u1x256_state_loongsonasx_t,
+                           nk_b128_vec_t, nk_dot_u1x256_init_loongsonasx, nk_load_b256_loongsonasx_,
+                           nk_partial_load_b1x256_serial_, nk_dot_u1x256_update_loongsonasx,
+                           nk_dot_u1x256_finalize_loongsonasx, nk_store_b128_loongsonasx_,
+                           nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/256, /*dimensions_per_value=*/8)
+nk_define_cross_packed_(dots, u1, loongsonasx, u1x8, u1x8, u32, nk_b256_vec_t, nk_dot_u1x256_state_loongsonasx_t,
+                        nk_b128_vec_t, nk_dot_u1x256_init_loongsonasx, nk_load_b256_loongsonasx_,
+                        nk_partial_load_b1x256_serial_, nk_load_b256_loongsonasx_, nk_partial_load_b1x256_serial_,
+                        nk_dot_u1x256_update_loongsonasx, nk_dot_u1x256_finalize_loongsonasx,
+                        nk_store_b128_loongsonasx_, nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/256, /*dimensions_per_value=*/8)
+
+/* BF16 GEMM: depth_simd_dimensions=16, packed stores raw bf16 (conversion is just an interleave).
+ *  Both symmetric and packed use the bf16x16 update which converts inline. */
+nk_define_cross_pack_size_(dots, bf16, loongsonasx, bf16, bf16, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/16,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, bf16, loongsonasx, bf16, bf16, nk_b256_vec_t, nk_load_b256_loongsonasx_,
+                      nk_partial_load_b16x16_serial_, nk_store_b256_loongsonasx_, nk_partial_store_b16x16_serial_,
+                      /*simd_width=*/16, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_bf16_,
+                      /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, bf16, loongsonasx, bf16, f32, nk_b256_vec_t, nk_dot_bf16x16_state_loongsonasx_t,
+                           nk_b128_vec_t, nk_dot_bf16x16_init_loongsonasx, nk_load_b256_loongsonasx_,
+                           nk_partial_load_b16x16_serial_, nk_dot_bf16x16_update_loongsonasx,
+                           nk_dot_bf16x16_finalize_loongsonasx, nk_store_b128_loongsonasx_,
+                           nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, bf16, loongsonasx, bf16, bf16, f32, nk_b256_vec_t, nk_dot_bf16x16_state_loongsonasx_t,
+                        nk_b128_vec_t, nk_dot_bf16x16_init_loongsonasx, nk_load_b256_loongsonasx_,
+                        nk_partial_load_b16x16_serial_, nk_load_b256_loongsonasx_, nk_partial_load_b16x16_serial_,
+                        nk_dot_bf16x16_update_loongsonasx, nk_dot_bf16x16_finalize_loongsonasx,
+                        nk_store_b128_loongsonasx_, nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+
+/* F16 GEMM: symmetric uses 256-bit raw f16 tiles (depth_simd_dimensions=16) with hardware xvfcvtl/xvfcvth,
+ *           packed pre-converts to f32 during packing (depth_simd_dimensions=8) since conversion is expensive. */
+nk_define_cross_pack_size_(dots, f16, loongsonasx, f16, f32, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/8,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, f16, loongsonasx, f16, f32, nk_b256_vec_t, nk_load_f16x8_to_f32x8_loongsonasx_,
+                      nk_partial_load_f16x8_to_f32x8_loongsonasx_, nk_store_b256_loongsonasx_,
+                      nk_partial_store_b32x8_serial_, /*simd_width=*/8, /*norm_value_type=*/f32,
+                      nk_dots_reduce_sumsq_f16_, /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, f16, loongsonasx, f16, f32, nk_b256_vec_t, nk_dot_f16x16_state_loongsonasx_t,
+                           nk_b128_vec_t, nk_dot_f16x16_init_loongsonasx, nk_load_b256_loongsonasx_,
+                           nk_partial_load_b16x16_serial_, nk_dot_f16x16_update_loongsonasx,
+                           nk_dot_f16x16_finalize_loongsonasx, nk_store_b128_loongsonasx_,
+                           nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, f16, loongsonasx, f16, f32, f32, nk_b256_vec_t, nk_dot_through_f32_state_loongsonasx_t_,
+                        nk_b128_vec_t, nk_dot_through_f32_init_loongsonasx_, nk_load_f16x8_to_f32x8_loongsonasx_,
+                        nk_partial_load_f16x8_to_f32x8_loongsonasx_, nk_load_b256_loongsonasx_,
+                        nk_partial_load_b32x8_serial_, nk_dot_through_f32_update_loongsonasx_,
+                        nk_dot_through_f32_finalize_loongsonasx_, nk_store_b128_loongsonasx_,
+                        nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_LOONGSONASX
+#endif // NK_TARGET_LOONGARCH_
+#endif // NK_DOTS_LOONGSONASX_H

package/include/numkong/dots/neon.h

@@ -28,9 +28,9 @@ extern "C" {
 /* F32 GEMM: depth_simd_dimensions=2 (2 f32s = 8 bytes = 64-bit input for f64 upcast accumulation) */
 nk_define_cross_pack_size_(dots, f32, neon, f32, f32, /*norm_value_type=*/f64, /*depth_simd_dimensions=*/2,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, f32, neon, f32, f32, nk_assign_from_to_, /*norm_value_type=*/f64, nk_dots_reduce_sumsq_f32_,
-                      /*depth_simd_dimensions=*/2,
-                      /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, f32, neon, f32, f32, nk_b128_vec_t, nk_load_b128_neon_, nk_partial_load_b32x4_serial_,
+                      nk_store_b128_neon_, nk_partial_store_b32x4_serial_, /*simd_width=*/4, /*norm_value_type=*/f64,
+                      nk_dots_reduce_sumsq_f32_, /*depth_simd_dimensions=*/2, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, f32, neon, f32, f64, nk_b64_vec_t, nk_dot_f32x2_state_neon_t, nk_b256_vec_t,
                            nk_dot_f32x2_init_neon, nk_load_b64_neon_, nk_partial_load_b32x2_serial_,
                            nk_dot_f32x2_update_neon, nk_dot_f32x2_finalize_neon, nk_store_b256_neon_,
@@ -45,8 +45,9 @@ nk_define_cross_packed_(dots, f32, neon, f32, f32, f64, nk_b64_vec_t, nk_dot_f32
 /* U1 GEMM: depth_simd_dimensions=128 (128 bits = 16 bytes = NEON register width) */
 nk_define_cross_pack_size_(dots, u1, neon, u1x8, u1x8, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/128,
                            /*dimensions_per_value=*/8)
-nk_define_cross_pack_(dots, u1, neon, u1x8, u1x8, nk_assign_from_to_, /*norm_value_type=*/u32, nk_dots_reduce_sum_u1_,
-                      /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/8)
+nk_define_cross_pack_(dots, u1, neon, u1x8, u1x8, nk_b128_vec_t, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                      nk_store_b128_neon_, nk_partial_store_b8x16_serial_, /*simd_width=*/16, /*norm_value_type=*/u32,
+                      nk_dots_reduce_sum_u1_, /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/8)
 nk_define_cross_symmetric_(dots, u1, neon, u1x8, u32, nk_b128_vec_t, nk_dot_u1x128_state_neon_t, nk_b128_vec_t,
                            nk_dot_u1x128_init_neon, nk_load_b128_neon_, nk_partial_load_b1x128_serial_,
                            nk_dot_u1x128_update_neon, nk_dot_u1x128_finalize_neon, nk_store_b128_neon_,
@@ -61,7 +62,8 @@ nk_define_cross_packed_(dots, u1, neon, u1x8, u1x8, u32, nk_b128_vec_t, nk_dot_u
 /* BF16 GEMM: depth_simd_dimensions=8 (8 bf16s = 16 bytes = NEON register width) */
 nk_define_cross_pack_size_(dots, bf16, neon, bf16, bf16, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/8,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, bf16, neon, bf16, bf16, nk_assign_from_to_, /*norm_value_type=*/f32,
+nk_define_cross_pack_(dots, bf16, neon, bf16, bf16, nk_b128_vec_t, nk_load_b128_neon_, nk_partial_load_b16x8_serial_,
+                      nk_store_b128_neon_, nk_partial_store_b16x8_serial_, /*simd_width=*/8, /*norm_value_type=*/f32,
                       nk_dots_reduce_sumsq_bf16_, /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, bf16, neon, bf16, f32, nk_b128_vec_t, nk_dot_bf16x8_state_neon_t, nk_b128_vec_t,
                            nk_dot_bf16x8_init_neon, nk_load_b128_neon_, nk_partial_load_b16x8_serial_,
@@ -76,8 +78,9 @@ nk_define_cross_packed_(dots, bf16, neon, bf16, bf16, f32, nk_b128_vec_t, nk_dot
 /* F16 GEMM: depth_simd_dimensions=8 (8 f16s = 16 bytes = NEON register width) */
 nk_define_cross_pack_size_(dots, f16, neon, f16, f16, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/8,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, f16, neon, f16, f16, nk_assign_from_to_, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_f16_,
-                      /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, f16, neon, f16, f16, nk_b128_vec_t, nk_load_b128_neon_, nk_partial_load_b16x8_serial_,
+                      nk_store_b128_neon_, nk_partial_store_b16x8_serial_, /*simd_width=*/8, /*norm_value_type=*/f32,
+                      nk_dots_reduce_sumsq_f16_, /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, f16, neon, f16, f32, nk_b128_vec_t, nk_dot_f16x8_state_neon_t, nk_b128_vec_t,
                            nk_dot_f16x8_init_neon, nk_load_b128_neon_, nk_partial_load_b16x8_serial_,
                            nk_dot_f16x8_update_neon, nk_dot_f16x8_finalize_neon, nk_store_b128_neon_,
@@ -91,9 +94,9 @@ nk_define_cross_packed_(dots, f16, neon, f16, f16, f32, nk_b128_vec_t, nk_dot_f1
 /* F64 GEMM: depth_simd_dimensions=2 (2 f64s = 16 bytes = NEON register width) */
 nk_define_cross_pack_size_(dots, f64, neon, f64, f64, /*norm_value_type=*/f64, /*depth_simd_dimensions=*/2,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, f64, neon, f64, f64, nk_assign_from_to_, /*norm_value_type=*/f64, nk_dots_reduce_sumsq_f64_,
-                      /*depth_simd_dimensions=*/2,
-                      /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, f64, neon, f64, f64, nk_b128_vec_t, nk_load_b128_neon_, nk_partial_load_b64x2_serial_,
+                      nk_store_b128_neon_, nk_partial_store_b64x2_serial_, /*simd_width=*/2, /*norm_value_type=*/f64,
+                      nk_dots_reduce_sumsq_f64_, /*depth_simd_dimensions=*/2, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, f64, neon, f64, f64, nk_b128_vec_t, nk_dot_f64x2_state_neon_t, nk_b256_vec_t,
                            nk_dot_f64x2_init_neon, nk_load_b128_neon_, nk_partial_load_b64x2_serial_,
                            nk_dot_f64x2_update_neon, nk_dot_f64x2_finalize_neon, nk_store_b256_neon_,

package/include/numkong/dots/neonbfdot.h

@@ -28,9 +28,10 @@ extern "C" {
 /* BF16 GEMM: depth_simd_dimensions=8 (8 bf16s = 16 bytes = NEON register width) */
 nk_define_cross_pack_size_(dots, bf16, neonbfdot, bf16, bf16, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/8,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, bf16, neonbfdot, bf16, bf16, nk_assign_from_to_, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_bf16_, /*depth_simd_dimensions=*/8,
-                      /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, bf16, neonbfdot, bf16, bf16, nk_b128_vec_t, nk_load_b128_neon_,
+                      nk_partial_load_b16x8_serial_, nk_store_b128_neon_, nk_partial_store_b16x8_serial_,
+                      /*simd_width=*/8, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_bf16_,
+                      /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, bf16, neonbfdot, bf16, f32, nk_b128_vec_t, nk_dot_bf16x8_state_neonbfdot_t,
                            nk_b128_vec_t, nk_dot_bf16x8_init_neonbfdot, nk_load_b128_neon_,
                            nk_partial_load_b16x8_serial_, nk_dot_bf16x8_update_neonbfdot,

package/include/numkong/dots/neonfhm.h

@@ -31,9 +31,9 @@ extern "C" {
 /* F16 GEMM using FMLAL: depth_simd_dimensions=8 (8 f16s = 16 bytes = NEON register width) */
 nk_define_cross_pack_size_(dots, f16, neonfhm, f16, f16, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/8,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, f16, neonfhm, f16, f16, nk_assign_from_to_, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_f16_, /*depth_simd_dimensions=*/8,
-                      /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, f16, neonfhm, f16, f16, nk_b128_vec_t, nk_load_b128_neon_, nk_partial_load_b16x8_serial_,
+                      nk_store_b128_neon_, nk_partial_store_b16x8_serial_, /*simd_width=*/8, /*norm_value_type=*/f32,
+                      nk_dots_reduce_sumsq_f16_, /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, f16, neonfhm, f16, f32, nk_b128_vec_t, nk_dot_f16x8_state_neonfhm_t, nk_b128_vec_t,
                            nk_dot_f16x8_init_neonfhm, nk_load_b128_neon_, nk_partial_load_b16x8_serial_,
                            nk_dot_f16x8_update_neonfhm, nk_dot_f16x8_finalize_neonfhm, nk_store_b128_neon_,
@@ -48,9 +48,10 @@ nk_define_cross_packed_(dots, f16, neonfhm, f16, f16, f32, nk_b128_vec_t, nk_dot
 /* E4M3 GEMM via FMLAL: depth_simd_dimensions=16 (16 e4m3s = 16 bytes) */
 nk_define_cross_pack_size_(dots, e4m3, neonfhm, e4m3, e4m3, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/16,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, e4m3, neonfhm, e4m3, e4m3, nk_assign_from_to_, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_e4m3_, /*depth_simd_dimensions=*/16,
-                      /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, e4m3, neonfhm, e4m3, e4m3, nk_b128_vec_t, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                      nk_store_b128_neon_, nk_partial_store_b8x16_serial_,
+                      /*simd_width=*/16, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e4m3_,
+                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, e4m3, neonfhm, e4m3, f32, nk_b128_vec_t, nk_dot_e4m3x16_state_neonfhm_t, nk_b128_vec_t,
                            nk_dot_e4m3x16_init_neonfhm, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
                            nk_dot_e4m3x16_update_neonfhm, nk_dot_e4m3x16_finalize_neonfhm, nk_store_b128_neon_,
@@ -65,9 +66,10 @@ nk_define_cross_packed_(dots, e4m3, neonfhm, e4m3, e4m3, f32, nk_b128_vec_t, nk_
 /* E5M2 GEMM via FMLAL: depth_simd_dimensions=16 (16 e5m2s = 16 bytes) */
 nk_define_cross_pack_size_(dots, e5m2, neonfhm, e5m2, e5m2, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/16,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, e5m2, neonfhm, e5m2, e5m2, nk_assign_from_to_, /*norm_value_type=*/f32,
-                      nk_dots_reduce_sumsq_e5m2_, /*depth_simd_dimensions=*/16,
-                      /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, e5m2, neonfhm, e5m2, e5m2, nk_b128_vec_t, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                      nk_store_b128_neon_, nk_partial_store_b8x16_serial_,
+                      /*simd_width=*/16, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e5m2_,
+                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, e5m2, neonfhm, e5m2, f32, nk_b128_vec_t, nk_dot_e5m2x16_state_neonfhm_t, nk_b128_vec_t,
                            nk_dot_e5m2x16_init_neonfhm, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
                            nk_dot_e5m2x16_update_neonfhm, nk_dot_e5m2x16_finalize_neonfhm, nk_store_b128_neon_,

package/include/numkong/dots/neonfp8.h

@@ -0,0 +1,99 @@
+/**
+ *  @brief SIMD-accelerated Batched Dot Products for NEON FP8DOT4.
+ *  @file include/numkong/dots/neonfp8.h
+ *  @author Ash Vardanian
+ *  @date March 23, 2026
+ *
+ *  @sa include/numkong/dots.h
+ *
+ *  Uses FDOT (FEAT_FP8DOT4) for native FP8 4-way dot products accumulating into FP32.
+ *  Each FDOT processes 16 FP8 elements (128-bit register) into 4 FP32 accumulators.
+ */
+#ifndef NK_DOTS_NEONFP8_H
+#define NK_DOTS_NEONFP8_H
+
+#if NK_TARGET_ARM_
+#if NK_TARGET_NEONFP8
+
+#include "numkong/dot/neonfp8.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=armv8-a+simd+fp8dot4"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=armv8-a+simd+fp8dot4")
+#endif
+
+nk_define_cross_pack_size_(dots, e4m3, neonfp8, e4m3, e4m3, f32, 16, 1)
+nk_define_cross_pack_(dots, e4m3, neonfp8, e4m3, e4m3, nk_b128_vec_t, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                      nk_store_b128_neon_, nk_partial_store_b8x16_serial_,
+                      /*simd_width=*/16, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e4m3_,
+                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, e4m3, neonfp8, e4m3, f32, nk_b128_vec_t, nk_dot_e4m3x16_state_neonfp8_t, nk_b128_vec_t,
+                           nk_dot_e4m3x16_init_neonfp8, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                           nk_dot_e4m3x16_update_neonfp8, nk_dot_e4m3x16_finalize_neonfp8, nk_store_b128_neon_,
+                           nk_partial_store_b32x4_serial_, 16, 1)
+nk_define_cross_packed_(dots, e4m3, neonfp8, e4m3, e4m3, f32, nk_b128_vec_t, nk_dot_e4m3x16_state_neonfp8_t,
+                        nk_b128_vec_t, nk_dot_e4m3x16_init_neonfp8, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                        nk_load_b128_neon_, nk_partial_load_b8x16_serial_, nk_dot_e4m3x16_update_neonfp8,
+                        nk_dot_e4m3x16_finalize_neonfp8, nk_store_b128_neon_, nk_partial_store_b32x4_serial_, 16, 1)
+
+nk_define_cross_pack_size_(dots, e5m2, neonfp8, e5m2, e5m2, f32, 16, 1)
+nk_define_cross_pack_(dots, e5m2, neonfp8, e5m2, e5m2, nk_b128_vec_t, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                      nk_store_b128_neon_, nk_partial_store_b8x16_serial_,
+                      /*simd_width=*/16, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e5m2_,
+                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, e5m2, neonfp8, e5m2, f32, nk_b128_vec_t, nk_dot_e5m2x16_state_neonfp8_t, nk_b128_vec_t,
+                           nk_dot_e5m2x16_init_neonfp8, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                           nk_dot_e5m2x16_update_neonfp8, nk_dot_e5m2x16_finalize_neonfp8, nk_store_b128_neon_,
+                           nk_partial_store_b32x4_serial_, 16, 1)
+nk_define_cross_packed_(dots, e5m2, neonfp8, e5m2, e5m2, f32, nk_b128_vec_t, nk_dot_e5m2x16_state_neonfp8_t,
+                        nk_b128_vec_t, nk_dot_e5m2x16_init_neonfp8, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                        nk_load_b128_neon_, nk_partial_load_b8x16_serial_, nk_dot_e5m2x16_update_neonfp8,
+                        nk_dot_e5m2x16_finalize_neonfp8, nk_store_b128_neon_, nk_partial_store_b32x4_serial_, 16, 1)
+
+nk_define_cross_pack_size_(dots, e2m3, neonfp8, e2m3, e2m3, f32, 16, 1)
+nk_define_cross_pack_(dots, e2m3, neonfp8, e2m3, e2m3, nk_b128_vec_t, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                      nk_store_b128_neon_, nk_partial_store_b8x16_serial_,
+                      /*simd_width=*/16, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e2m3_,
+                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, e2m3, neonfp8, e2m3, f32, nk_b128_vec_t, nk_dot_e2m3x16_state_neonfp8_t, nk_b128_vec_t,
+                           nk_dot_e2m3x16_init_neonfp8, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                           nk_dot_e2m3x16_update_neonfp8, nk_dot_e2m3x16_finalize_neonfp8, nk_store_b128_neon_,
+                           nk_partial_store_b32x4_serial_, 16, 1)
+nk_define_cross_packed_(dots, e2m3, neonfp8, e2m3, e2m3, f32, nk_b128_vec_t, nk_dot_e2m3x16_state_neonfp8_t,
+                        nk_b128_vec_t, nk_dot_e2m3x16_init_neonfp8, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                        nk_load_b128_neon_, nk_partial_load_b8x16_serial_, nk_dot_e2m3x16_update_neonfp8,
+                        nk_dot_e2m3x16_finalize_neonfp8, nk_store_b128_neon_, nk_partial_store_b32x4_serial_, 16, 1)
+
+nk_define_cross_pack_size_(dots, e3m2, neonfp8, e3m2, e3m2, f32, 16, 1)
+nk_define_cross_pack_(dots, e3m2, neonfp8, e3m2, e3m2, nk_b128_vec_t, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                      nk_store_b128_neon_, nk_partial_store_b8x16_serial_,
+                      /*simd_width=*/16, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e3m2_,
+                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, e3m2, neonfp8, e3m2, f32, nk_b128_vec_t, nk_dot_e3m2x16_state_neonfp8_t, nk_b128_vec_t,
+                           nk_dot_e3m2x16_init_neonfp8, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                           nk_dot_e3m2x16_update_neonfp8, nk_dot_e3m2x16_finalize_neonfp8, nk_store_b128_neon_,
+                           nk_partial_store_b32x4_serial_, 16, 1)
+nk_define_cross_packed_(dots, e3m2, neonfp8, e3m2, e3m2, f32, nk_b128_vec_t, nk_dot_e3m2x16_state_neonfp8_t,
+                        nk_b128_vec_t, nk_dot_e3m2x16_init_neonfp8, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                        nk_load_b128_neon_, nk_partial_load_b8x16_serial_, nk_dot_e3m2x16_update_neonfp8,
+                        nk_dot_e3m2x16_finalize_neonfp8, nk_store_b128_neon_, nk_partial_store_b32x4_serial_, 16, 1)
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_NEONFP8
+#endif // NK_TARGET_ARM_
+#endif // NK_DOTS_NEONFP8_H

package/include/numkong/dots/neonsdot.h

@@ -28,9 +28,9 @@ extern "C" {
 /* I8 GEMM: depth_simd_dimensions=16 (16 i8s = 16 bytes = NEON register width) */
 nk_define_cross_pack_size_(dots, i8, neonsdot, i8, i8, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/16,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, i8, neonsdot, i8, i8, nk_assign_from_to_, /*norm_value_type=*/u32, nk_dots_reduce_sumsq_i8_,
-                      /*depth_simd_dimensions=*/16,
-                      /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, i8, neonsdot, i8, i8, nk_b128_vec_t, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                      nk_store_b128_neon_, nk_partial_store_b8x16_serial_, /*simd_width=*/16, /*norm_value_type=*/u32,
+                      nk_dots_reduce_sumsq_i8_, /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, i8, neonsdot, i8, i32, nk_b128_vec_t, nk_dot_i8x16_state_neonsdot_t, nk_b128_vec_t,
                            nk_dot_i8x16_init_neonsdot, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
                            nk_dot_i8x16_update_neonsdot, nk_dot_i8x16_finalize_neonsdot, nk_store_b128_neon_,
@@ -45,9 +45,9 @@ nk_define_cross_packed_(dots, i8, neonsdot, i8, i8, i32, nk_b128_vec_t, nk_dot_i
 /* U8 GEMM: depth_simd_dimensions=16 (16 u8s = 16 bytes = NEON register width) */
 nk_define_cross_pack_size_(dots, u8, neonsdot, u8, u8, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/16,
                            /*dimensions_per_value=*/1)
-nk_define_cross_pack_(dots, u8, neonsdot, u8, u8, nk_assign_from_to_, /*norm_value_type=*/u32, nk_dots_reduce_sumsq_u8_,
-                      /*depth_simd_dimensions=*/16,
-                      /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, u8, neonsdot, u8, u8, nk_b128_vec_t, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                      nk_store_b128_neon_, nk_partial_store_b8x16_serial_, /*simd_width=*/16, /*norm_value_type=*/u32,
+                      nk_dots_reduce_sumsq_u8_, /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
 nk_define_cross_symmetric_(dots, u8, neonsdot, u8, u32, nk_b128_vec_t, nk_dot_u8x16_state_neonsdot_t, nk_b128_vec_t,
                            nk_dot_u8x16_init_neonsdot, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
                            nk_dot_u8x16_update_neonsdot, nk_dot_u8x16_finalize_neonsdot, nk_store_b128_neon_,
@@ -62,9 +62,9 @@ nk_define_cross_packed_(dots, u8, neonsdot, u8, u8, u32, nk_b128_vec_t, nk_dot_u
 /* I4 GEMM: depth_simd_dimensions=32 (32 nibbles = 16 bytes = NEON register width) */
 nk_define_cross_pack_size_(dots, i4, neonsdot, i4x2, i4x2, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/32,
                            /*dimensions_per_value=*/2)
-nk_define_cross_pack_(dots, i4, neonsdot, i4x2, i4x2, nk_assign_from_to_, /*norm_value_type=*/u32,
-                      nk_dots_reduce_sumsq_i4_, /*depth_simd_dimensions=*/32,
-                      /*dimensions_per_value=*/2)
+nk_define_cross_pack_(dots, i4, neonsdot, i4x2, i4x2, nk_b128_vec_t, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                      nk_store_b128_neon_, nk_partial_store_b8x16_serial_, /*simd_width=*/16, /*norm_value_type=*/u32,
+                      nk_dots_reduce_sumsq_i4_, /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/2)
 nk_define_cross_symmetric_(dots, i4, neonsdot, i4x2, i32, nk_b128_vec_t, nk_dot_i4x32_state_neonsdot_t, nk_b128_vec_t,
                            nk_dot_i4x32_init_neonsdot, nk_load_b128_neon_, nk_partial_load_b4x32_serial_,
                            nk_dot_i4x32_update_neonsdot, nk_dot_i4x32_finalize_neonsdot, nk_store_b128_neon_,
@@ -79,9 +79,9 @@ nk_define_cross_packed_(dots, i4, neonsdot, i4x2, i4x2, i32, nk_b128_vec_t, nk_d
 /* U4 GEMM: depth_simd_dimensions=32 (32 nibbles = 16 bytes = NEON register width) */
 nk_define_cross_pack_size_(dots, u4, neonsdot, u4x2, u4x2, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/32,
                            /*dimensions_per_value=*/2)
-nk_define_cross_pack_(dots, u4, neonsdot, u4x2, u4x2, nk_assign_from_to_, /*norm_value_type=*/u32,
-                      nk_dots_reduce_sumsq_u4_, /*depth_simd_dimensions=*/32,
-                      /*dimensions_per_value=*/2)
+nk_define_cross_pack_(dots, u4, neonsdot, u4x2, u4x2, nk_b128_vec_t, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                      nk_store_b128_neon_, nk_partial_store_b8x16_serial_, /*simd_width=*/16, /*norm_value_type=*/u32,
+                      nk_dots_reduce_sumsq_u4_, /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/2)
 nk_define_cross_symmetric_(dots, u4, neonsdot, u4x2, u32, nk_b128_vec_t, nk_dot_u4x32_state_neonsdot_t, nk_b128_vec_t,
                            nk_dot_u4x32_init_neonsdot, nk_load_b128_neon_, nk_partial_load_b4x32_serial_,
                            nk_dot_u4x32_update_neonsdot, nk_dot_u4x32_finalize_neonsdot, nk_store_b128_neon_,
@@ -93,6 +93,42 @@ nk_define_cross_packed_(dots, u4, neonsdot, u4x2, u4x2, u32, nk_b128_vec_t, nk_d
                         nk_dot_u4x32_finalize_neonsdot, nk_store_b128_neon_, nk_partial_store_b32x4_serial_,
                         /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/2)
 
+/* E2M3: depth_simd_dimensions=16 (16 e2m3 values = 16 bytes = NEON register width) */
+nk_define_cross_pack_size_(dots, e2m3, neonsdot, e2m3, e2m3, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/16,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, e2m3, neonsdot, e2m3, e2m3, nk_b128_vec_t, nk_load_b128_neon_,
+                      nk_partial_load_b8x16_serial_, nk_store_b128_neon_, nk_partial_store_b8x16_serial_,
+                      /*simd_width=*/16, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e2m3_,
+                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, e2m3, neonsdot, e2m3, f32, nk_b128_vec_t, nk_dot_e2m3x16_state_neonsdot_t,
+                           nk_b128_vec_t, nk_dot_e2m3x16_init_neonsdot, nk_load_b128_neon_,
+                           nk_partial_load_b8x16_serial_, nk_dot_e2m3x16_update_neonsdot,
+                           nk_dot_e2m3x16_finalize_neonsdot, nk_store_b128_neon_, nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, e2m3, neonsdot, e2m3, e2m3, f32, nk_b128_vec_t, nk_dot_e2m3x16_state_neonsdot_t,
+                        nk_b128_vec_t, nk_dot_e2m3x16_init_neonsdot, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                        nk_load_b128_neon_, nk_partial_load_b8x16_serial_, nk_dot_e2m3x16_update_neonsdot,
+                        nk_dot_e2m3x16_finalize_neonsdot, nk_store_b128_neon_, nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+
+/* E3M2: depth_simd_dimensions=16 (16 e3m2 values = 16 bytes = NEON register width) */
+nk_define_cross_pack_size_(dots, e3m2, neonsdot, e3m2, e3m2, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/16,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, e3m2, neonsdot, e3m2, e3m2, nk_b128_vec_t, nk_load_b128_neon_,
+                      nk_partial_load_b8x16_serial_, nk_store_b128_neon_, nk_partial_store_b8x16_serial_,
+                      /*simd_width=*/16, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_e3m2_,
+                      /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, e3m2, neonsdot, e3m2, f32, nk_b128_vec_t, nk_dot_e3m2x16_state_neonsdot_t,
+                           nk_b128_vec_t, nk_dot_e3m2x16_init_neonsdot, nk_load_b128_neon_,
+                           nk_partial_load_b8x16_serial_, nk_dot_e3m2x16_update_neonsdot,
+                           nk_dot_e3m2x16_finalize_neonsdot, nk_store_b128_neon_, nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, e3m2, neonsdot, e3m2, e3m2, f32, nk_b128_vec_t, nk_dot_e3m2x16_state_neonsdot_t,
+                        nk_b128_vec_t, nk_dot_e3m2x16_init_neonsdot, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                        nk_load_b128_neon_, nk_partial_load_b8x16_serial_, nk_dot_e3m2x16_update_neonsdot,
+                        nk_dot_e3m2x16_finalize_neonsdot, nk_store_b128_neon_, nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+
 #if defined(__clang__)
 #pragma clang attribute pop
 #elif defined(__GNUC__)