numkong 7.0.0

package/include/numkong/dots/icelake.h

@@ -0,0 +1,171 @@
+/**
+ *  @brief SIMD-accelerated Batched Dot Products for Ice Lake.
+ *  @file include/numkong/dots/icelake.h
+ *  @author Ash Vardanian
+ *  @date December 27, 2025
+ *
+ *  @sa include/numkong/dots.h
+ *
+ *  @section ice_dots_instructions Relevant Instructions
+ *
+ *      Intrinsic                   Instruction                     Ice         Genoa
+ *      _mm512_dpbusd_epi32         VPDPBUSD (ZMM, ZMM, ZMM)        5cy @ p0    4cy @ p01
+ *      _mm512_dpwssd_epi32         VPDPWSSD (ZMM, ZMM, ZMM)        5cy @ p0    4cy @ p01
+ *      _mm512_cvtepi8_epi32        VPMOVSXBD (ZMM, XMM)            3cy @ p5    3cy @ p12
+ *      _mm512_loadu_si512          VMOVDQU64 (ZMM, M512)           7cy @ p23   7cy @ p23
+ *
+ *  Ice Lake's VNNI instructions accelerate int8 GEMM by computing 4-element dot products per lane.
+ *  VPDPBUSD/VPDPWSSD bottleneck on port 0, limiting throughput to 1/cy. AMD Genoa achieves 0.5/cy
+ *  via dual-issue on ports 0-1, making it significantly faster for quantized inference workloads.
+ */
+#ifndef NK_DOTS_ICELAKE_H
+#define NK_DOTS_ICELAKE_H
+
+#if NK_TARGET_X86_
+#if NK_TARGET_ICELAKE
+
+#include "numkong/dot/icelake.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(                                                                                        \
+    __attribute__((                                                                                                  \
+        target("avx2,avx512f,avx512vl,avx512bw,avx512dq,avx512vnni,avx512vbmi,avx512vpopcntdq,f16c,fma,bmi,bmi2"))), \
+    apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "avx512dq", "avx512vnni", "avx512vbmi", \
+                   "avx512vpopcntdq", "f16c", "fma", "bmi", "bmi2")
+#endif
+
+/* I8 GEMM: depth_simd_dimensions=64 — compensated (B sums precomputed in pack) */
+nk_define_cross_compensated_pack_size_(dots, i8, icelake, i8, i8,
+                                       /*sum_value_type=*/i32, /*norm_value_type=*/u32,
+                                       /*depth_simd_dimensions=*/64, /*dimensions_per_value=*/1)
+nk_define_cross_compensated_pack_(dots, i8, icelake, i8, i8, nk_assign_from_to_,
+                                  /*sum_value_type=*/i32, /*norm_value_type=*/u32, nk_dots_reduce_moments_i8_,
+                                  /*depth_simd_dimensions=*/64, /*dimensions_per_value=*/1)
+nk_define_cross_compensated_symmetric_(dots, i8, icelake, i8, i32,
+                                       /*sum_value_type=*/i32, /*norm_value_type=*/u32, nk_b512_vec_t,
+                                       nk_dot_i8x64_state_icelake_t, nk_b128_vec_t, nk_dot_i8x64_init_icelake,
+                                       nk_load_b512_skylake_, nk_partial_load_b8x64_skylake_,
+                                       nk_dot_i8x64_update_icelake, nk_dot_i8x64_finalize_icelake,
+                                       nk_store_b128_haswell_, nk_partial_store_b32x4_skylake_, nk_load_b128_haswell_,
+                                       nk_partial_load_b32x4_skylake_, nk_sum_i8x64_state_icelake_t,
+                                       nk_sum_i8x64_init_icelake, nk_sum_i8x64_update_icelake,
+                                       nk_sum_i8x64_finalize_icelake,
+                                       /*depth_simd_dimensions=*/64, /*dimensions_per_value=*/1)
+nk_define_cross_compensated_packed_(dots, i8, icelake, i8, i8, i32,
+                                    /*sum_value_type=*/i32, /*norm_value_type=*/u32, nk_b512_vec_t,
+                                    nk_dot_i8x64_state_icelake_t, nk_b128_vec_t, nk_dot_i8x64_init_icelake,
+                                    nk_load_b512_skylake_, nk_partial_load_b8x64_skylake_, nk_load_b512_skylake_,
+                                    nk_partial_load_b8x64_skylake_, nk_dot_i8x64_update_icelake,
+                                    nk_dot_i8x64_finalize_icelake, nk_store_b128_haswell_,
+                                    nk_partial_store_b32x4_skylake_, nk_load_b128_haswell_,
+                                    nk_partial_load_b32x4_skylake_, nk_dots_reduce_sum_i8_stub_,
+                                    /*depth_simd_dimensions=*/64, /*dimensions_per_value=*/1)
+
+/* U8 GEMM: depth_simd_dimensions=64 — compensated (operand swap, B sums precomputed) */
+nk_define_cross_compensated_pack_size_(dots, u8, icelake, u8, u8,
+                                       /*sum_value_type=*/u32, /*norm_value_type=*/u32,
+                                       /*depth_simd_dimensions=*/64, /*dimensions_per_value=*/1)
+nk_define_cross_compensated_pack_(dots, u8, icelake, u8, u8, nk_assign_from_to_,
+                                  /*sum_value_type=*/u32, /*norm_value_type=*/u32, nk_dots_reduce_moments_u8_,
+                                  /*depth_simd_dimensions=*/64, /*dimensions_per_value=*/1)
+nk_define_cross_compensated_symmetric_(dots, u8, icelake, u8, u32,
+                                       /*sum_value_type=*/u32, /*norm_value_type=*/u32, nk_b512_vec_t,
+                                       nk_dot_u8x64_state_icelake_t, nk_b128_vec_t, nk_dot_u8x64_init_icelake,
+                                       nk_load_b512_skylake_, nk_partial_load_b8x64_skylake_,
+                                       nk_dot_u8x64_update_icelake, nk_dot_u8x64_finalize_icelake,
+                                       nk_store_b128_haswell_, nk_partial_store_b32x4_skylake_, nk_load_b128_haswell_,
+                                       nk_partial_load_b32x4_skylake_, nk_sum_u8x64_state_icelake_t,
+                                       nk_sum_u8x64_init_icelake, nk_sum_u8x64_update_icelake,
+                                       nk_sum_u8x64_finalize_icelake,
+                                       /*depth_simd_dimensions=*/64, /*dimensions_per_value=*/1)
+nk_define_cross_compensated_packed_(dots, u8, icelake, u8, u8, u32,
+                                    /*sum_value_type=*/u32, /*norm_value_type=*/u32, nk_b512_vec_t,
+                                    nk_dot_u8x64_state_icelake_t, nk_b128_vec_t, nk_dot_u8x64_init_icelake,
+                                    nk_load_b512_skylake_, nk_partial_load_b8x64_skylake_, nk_load_b512_skylake_,
+                                    nk_partial_load_b8x64_skylake_, nk_dot_u8x64_update_icelake,
+                                    nk_dot_u8x64_finalize_icelake, nk_store_b128_haswell_,
+                                    nk_partial_store_b32x4_skylake_, nk_load_b128_haswell_,
+                                    nk_partial_load_b32x4_skylake_, nk_dots_reduce_sum_u8_stub_,
+                                    /*depth_simd_dimensions=*/64, /*dimensions_per_value=*/1)
+
+/* I4 GEMM: depth_simd_dimensions=128 — compensated (A+B sums precomputed) */
+nk_define_cross_compensated_pack_size_(dots, i4, icelake, i4x2, i4x2,
+                                       /*sum_value_type=*/i32, /*norm_value_type=*/u32,
+                                       /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/2)
+nk_define_cross_compensated_pack_(dots, i4, icelake, i4x2, i4x2, nk_assign_from_to_,
+                                  /*sum_value_type=*/i32, /*norm_value_type=*/u32, nk_dots_reduce_moments_i4_,
+                                  /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/2)
+nk_define_cross_compensated_symmetric_(dots, i4, icelake, i4x2, i32,
+                                       /*sum_value_type=*/i32, /*norm_value_type=*/u32, nk_b512_vec_t,
+                                       nk_dot_i4x128_state_icelake_t, nk_b128_vec_t, nk_dot_i4x128_init_icelake,
+                                       nk_load_b512_skylake_, nk_partial_load_b4x128_skylake_,
+                                       nk_dot_i4x128_update_icelake, nk_dot_i4x128_finalize_icelake,
+                                       nk_store_b128_haswell_, nk_partial_store_b32x4_skylake_, nk_load_b128_haswell_,
+                                       nk_partial_load_b32x4_skylake_, nk_sum_i4x128_state_icelake_t,
+                                       nk_sum_i4x128_init_icelake, nk_sum_i4x128_update_icelake,
+                                       nk_sum_i4x128_finalize_icelake,
+                                       /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/2)
+nk_define_cross_compensated_packed_(dots, i4, icelake, i4x2, i4x2, i32,
+                                    /*sum_value_type=*/i32, /*norm_value_type=*/u32, nk_b512_vec_t,
+                                    nk_dot_i4x128_state_icelake_t, nk_b128_vec_t, nk_dot_i4x128_init_icelake,
+                                    nk_load_b512_skylake_, nk_partial_load_b4x128_skylake_, nk_load_b512_skylake_,
+                                    nk_partial_load_b4x128_skylake_, nk_dot_i4x128_update_icelake,
+                                    nk_dot_i4x128_finalize_icelake, nk_store_b128_haswell_,
+                                    nk_partial_store_b32x4_skylake_, nk_load_b128_haswell_,
+                                    nk_partial_load_b32x4_skylake_, nk_dots_reduce_sum_i4_,
+                                    /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/2)
+
+/* U4 GEMM: depth_simd_dimensions=128 (128 nibbles = 64 bytes = full cache line) */
+nk_define_cross_pack_size_(dots, u4, icelake, u4x2, u4x2, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/128,
+                           /*dimensions_per_value=*/2)
+nk_define_cross_pack_(dots, u4, icelake, u4x2, u4x2, nk_assign_from_to_, /*norm_value_type=*/u32,
+                      nk_dots_reduce_sumsq_u4_, /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/2)
+
+nk_define_cross_symmetric_(dots, u4, icelake, u4x2, u32, nk_b512_vec_t, nk_dot_u4x128_state_icelake_t, nk_b128_vec_t,
+                           nk_dot_u4x128_init_icelake, nk_load_b512_skylake_, nk_partial_load_b4x128_skylake_,
+                           nk_dot_u4x128_update_icelake, nk_dot_u4x128_finalize_icelake, nk_store_b128_haswell_,
+                           nk_partial_store_b32x4_skylake_,
+                           /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/2)
+nk_define_cross_packed_(dots, u4, icelake, u4x2, u4x2, u32, nk_b512_vec_t, nk_dot_u4x128_state_icelake_t, nk_b128_vec_t,
+                        nk_dot_u4x128_init_icelake, nk_load_b512_skylake_, nk_partial_load_b4x128_skylake_,
+                        nk_load_b512_skylake_, nk_partial_load_b4x128_skylake_, nk_dot_u4x128_update_icelake,
+                        nk_dot_u4x128_finalize_icelake, nk_store_b128_haswell_, nk_partial_store_b32x4_skylake_,
+                        /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/2)
+
+/* U1 GEMM: depth_simd_dimensions=512 (512 bits = 64 bytes = full cache line) */
+nk_define_cross_pack_size_(dots, u1, icelake, u1x8, u1x8, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/512,
+                           /*dimensions_per_value=*/8)
+nk_define_cross_pack_(dots, u1, icelake, u1x8, u1x8, nk_assign_from_to_, /*norm_value_type=*/u32,
+                      nk_dots_reduce_sum_u1_,
+                      /*depth_simd_dimensions=*/512, /*dimensions_per_value=*/8)
+nk_define_cross_symmetric_(dots, u1, icelake, u1x8, u32, nk_b512_vec_t, nk_dot_u1x512_state_icelake_t, nk_b128_vec_t,
+                           nk_dot_u1x512_init_icelake, nk_load_b512_skylake_, nk_partial_load_b1x512_skylake_,
+                           nk_dot_u1x512_update_icelake, nk_dot_u1x512_finalize_icelake, nk_store_b128_haswell_,
+                           nk_partial_store_b32x4_skylake_,
+                           /*depth_simd_dimensions=*/512, /*dimensions_per_value=*/8)
+nk_define_cross_packed_(dots, u1, icelake, u1x8, u1x8, u32, nk_b512_vec_t, nk_dot_u1x512_state_icelake_t, nk_b128_vec_t,
+                        nk_dot_u1x512_init_icelake, nk_load_b512_skylake_, nk_partial_load_b1x512_skylake_,
+                        nk_load_b512_skylake_, nk_partial_load_b1x512_skylake_, nk_dot_u1x512_update_icelake,
+                        nk_dot_u1x512_finalize_icelake, nk_store_b128_haswell_, nk_partial_store_b32x4_skylake_,
+                        /*depth_simd_dimensions=*/512, /*dimensions_per_value=*/8)
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_ICELAKE
+#endif // NK_TARGET_X86_
+#endif // NK_DOTS_ICELAKE_H

package/include/numkong/dots/neon.h

@@ -0,0 +1,120 @@
+/**
+ *  @brief SIMD-accelerated Batched Dot Products for NEON.
+ *  @file include/numkong/dots/neon.h
+ *  @author Ash Vardanian
+ *  @date December 27, 2025
+ *
+ *  @sa include/numkong/dots.h
+ */
+#ifndef NK_DOTS_NEON_H
+#define NK_DOTS_NEON_H
+
+#if NK_TARGET_ARM_
+#if NK_TARGET_NEON
+
+#include "numkong/dot/neon.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=armv8-a+simd"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=armv8-a+simd")
+#endif
+
+/* F32 GEMM: depth_simd_dimensions=2 (2 f32s = 8 bytes = 64-bit input for f64 upcast accumulation) */
+nk_define_cross_pack_size_(dots, f32, neon, f32, f32, /*norm_value_type=*/f64, /*depth_simd_dimensions=*/2,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, f32, neon, f32, f32, nk_assign_from_to_, /*norm_value_type=*/f64, nk_dots_reduce_sumsq_f32_,
+                      /*depth_simd_dimensions=*/2,
+                      /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, f32, neon, f32, f64, nk_b64_vec_t, nk_dot_f32x2_state_neon_t, nk_b256_vec_t,
+                           nk_dot_f32x2_init_neon, nk_load_b64_neon_, nk_partial_load_b32x2_serial_,
+                           nk_dot_f32x2_update_neon, nk_dot_f32x2_finalize_neon, nk_store_b256_neon_,
+                           nk_partial_store_b64x4_serial_,
+                           /*depth_simd_dimensions=*/2, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, f32, neon, f32, f32, f64, nk_b64_vec_t, nk_dot_f32x2_state_neon_t, nk_b256_vec_t,
+                        nk_dot_f32x2_init_neon, nk_load_b64_neon_, nk_partial_load_b32x2_serial_, nk_load_b64_neon_,
+                        nk_partial_load_b32x2_serial_, nk_dot_f32x2_update_neon, nk_dot_f32x2_finalize_neon,
+                        nk_store_b256_neon_, nk_partial_store_b64x4_serial_,
+                        /*depth_simd_dimensions=*/2, /*dimensions_per_value=*/1)
+
+/* U1 GEMM: depth_simd_dimensions=128 (128 bits = 16 bytes = NEON register width) */
+nk_define_cross_pack_size_(dots, u1, neon, u1x8, u1x8, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/128,
+                           /*dimensions_per_value=*/8)
+nk_define_cross_pack_(dots, u1, neon, u1x8, u1x8, nk_assign_from_to_, /*norm_value_type=*/u32, nk_dots_reduce_sum_u1_,
+                      /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/8)
+nk_define_cross_symmetric_(dots, u1, neon, u1x8, u32, nk_b128_vec_t, nk_dot_u1x128_state_neon_t, nk_b128_vec_t,
+                           nk_dot_u1x128_init_neon, nk_load_b128_neon_, nk_partial_load_b1x128_serial_,
+                           nk_dot_u1x128_update_neon, nk_dot_u1x128_finalize_neon, nk_store_b128_neon_,
+                           nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/8)
+nk_define_cross_packed_(dots, u1, neon, u1x8, u1x8, u32, nk_b128_vec_t, nk_dot_u1x128_state_neon_t, nk_b128_vec_t,
+                        nk_dot_u1x128_init_neon, nk_load_b128_neon_, nk_partial_load_b1x128_serial_, nk_load_b128_neon_,
+                        nk_partial_load_b1x128_serial_, nk_dot_u1x128_update_neon, nk_dot_u1x128_finalize_neon,
+                        nk_store_b128_neon_, nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/128, /*dimensions_per_value=*/8)
+
+/* BF16 GEMM: depth_simd_dimensions=8 (8 bf16s = 16 bytes = NEON register width) */
+nk_define_cross_pack_size_(dots, bf16, neon, bf16, bf16, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/8,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, bf16, neon, bf16, bf16, nk_assign_from_to_, /*norm_value_type=*/f32,
+                      nk_dots_reduce_sumsq_bf16_, /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, bf16, neon, bf16, f32, nk_b128_vec_t, nk_dot_bf16x8_state_neon_t, nk_b128_vec_t,
+                           nk_dot_bf16x8_init_neon, nk_load_b128_neon_, nk_partial_load_b16x8_serial_,
+                           nk_dot_bf16x8_update_neon, nk_dot_bf16x8_finalize_neon, nk_store_b128_neon_,
+                           nk_partial_store_b32x4_serial_, /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, bf16, neon, bf16, bf16, f32, nk_b128_vec_t, nk_dot_bf16x8_state_neon_t, nk_b128_vec_t,
+                        nk_dot_bf16x8_init_neon, nk_load_b128_neon_, nk_partial_load_b16x8_serial_, nk_load_b128_neon_,
+                        nk_partial_load_b16x8_serial_, nk_dot_bf16x8_update_neon, nk_dot_bf16x8_finalize_neon,
+                        nk_store_b128_neon_, nk_partial_store_b32x4_serial_, /*depth_simd_dimensions=*/8,
+                        /*dimensions_per_value=*/1)
+
+/* F16 GEMM: depth_simd_dimensions=8 (8 f16s = 16 bytes = NEON register width) */
+nk_define_cross_pack_size_(dots, f16, neon, f16, f16, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/8,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, f16, neon, f16, f16, nk_assign_from_to_, /*norm_value_type=*/f32, nk_dots_reduce_sumsq_f16_,
+                      /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, f16, neon, f16, f32, nk_b128_vec_t, nk_dot_f16x8_state_neon_t, nk_b128_vec_t,
+                           nk_dot_f16x8_init_neon, nk_load_b128_neon_, nk_partial_load_b16x8_serial_,
+                           nk_dot_f16x8_update_neon, nk_dot_f16x8_finalize_neon, nk_store_b128_neon_,
+                           nk_partial_store_b32x4_serial_, /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, f16, neon, f16, f16, f32, nk_b128_vec_t, nk_dot_f16x8_state_neon_t, nk_b128_vec_t,
+                        nk_dot_f16x8_init_neon, nk_load_b128_neon_, nk_partial_load_b16x8_serial_, nk_load_b128_neon_,
+                        nk_partial_load_b16x8_serial_, nk_dot_f16x8_update_neon, nk_dot_f16x8_finalize_neon,
+                        nk_store_b128_neon_, nk_partial_store_b32x4_serial_, /*depth_simd_dimensions=*/8,
+                        /*dimensions_per_value=*/1)
+
+/* F64 GEMM: depth_simd_dimensions=2 (2 f64s = 16 bytes = NEON register width) */
+nk_define_cross_pack_size_(dots, f64, neon, f64, f64, /*norm_value_type=*/f64, /*depth_simd_dimensions=*/2,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, f64, neon, f64, f64, nk_assign_from_to_, /*norm_value_type=*/f64, nk_dots_reduce_sumsq_f64_,
+                      /*depth_simd_dimensions=*/2,
+                      /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, f64, neon, f64, f64, nk_b128_vec_t, nk_dot_f64x2_state_neon_t, nk_b256_vec_t,
+                           nk_dot_f64x2_init_neon, nk_load_b128_neon_, nk_partial_load_b64x2_serial_,
+                           nk_dot_f64x2_update_neon, nk_dot_f64x2_finalize_neon, nk_store_b256_neon_,
+                           nk_partial_store_b64x4_serial_,
+                           /*depth_simd_dimensions=*/2, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, f64, neon, f64, f64, f64, nk_b128_vec_t, nk_dot_f64x2_state_neon_t, nk_b256_vec_t,
+                        nk_dot_f64x2_init_neon, nk_load_b128_neon_, nk_partial_load_b64x2_serial_, nk_load_b128_neon_,
+                        nk_partial_load_b64x2_serial_, nk_dot_f64x2_update_neon, nk_dot_f64x2_finalize_neon,
+                        nk_store_b256_neon_, nk_partial_store_b64x4_serial_,
+                        /*depth_simd_dimensions=*/2, /*dimensions_per_value=*/1)
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_NEON
+#endif // NK_TARGET_ARM_
+#endif // NK_DOTS_NEON_H

package/include/numkong/dots/neonbfdot.h

@@ -0,0 +1,58 @@
+/**
+ *  @brief SIMD-accelerated Batched Dot Products for NEON BF16.
+ *  @file include/numkong/dots/neonbfdot.h
+ *  @author Ash Vardanian
+ *  @date December 27, 2025
+ *
+ *  @sa include/numkong/dots.h
+ */
+#ifndef NK_DOTS_NEONBFDOT_H
+#define NK_DOTS_NEONBFDOT_H
+
+#if NK_TARGET_ARM_
+#if NK_TARGET_NEONBFDOT
+
+#include "numkong/dot/neonbfdot.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=armv8.6-a+simd+bf16"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=armv8.6-a+simd+bf16")
+#endif
+
+/* BF16 GEMM: depth_simd_dimensions=8 (8 bf16s = 16 bytes = NEON register width) */
+nk_define_cross_pack_size_(dots, bf16, neonbfdot, bf16, bf16, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/8,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, bf16, neonbfdot, bf16, bf16, nk_assign_from_to_, /*norm_value_type=*/f32,
+                      nk_dots_reduce_sumsq_bf16_, /*depth_simd_dimensions=*/8,
+                      /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, bf16, neonbfdot, bf16, f32, nk_b128_vec_t, nk_dot_bf16x8_state_neonbfdot_t,
+                           nk_b128_vec_t, nk_dot_bf16x8_init_neonbfdot, nk_load_b128_neon_,
+                           nk_partial_load_b16x8_serial_, nk_dot_bf16x8_update_neonbfdot,
+                           nk_dot_bf16x8_finalize_neonbfdot, nk_store_b128_neon_, nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, bf16, neonbfdot, bf16, bf16, f32, nk_b128_vec_t, nk_dot_bf16x8_state_neonbfdot_t,
+                        nk_b128_vec_t, nk_dot_bf16x8_init_neonbfdot, nk_load_b128_neon_, nk_partial_load_b16x8_serial_,
+                        nk_load_b128_neon_, nk_partial_load_b16x8_serial_, nk_dot_bf16x8_update_neonbfdot,
+                        nk_dot_bf16x8_finalize_neonbfdot, nk_store_b128_neon_, nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/8,
+                        /*dimensions_per_value=*/1)
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_NEONBFDOT
+#endif // NK_TARGET_ARM_
+#endif // NK_DOTS_NEONBFDOT_H

package/include/numkong/dots/neonfhm.h

@@ -0,0 +1,94 @@
+/**
+ *  @brief SIMD-accelerated Batched Dot Products for NEON FHM.
+ *  @file include/numkong/dots/neonfhm.h
+ *  @author Ash Vardanian
+ *  @date December 28, 2025
+ *
+ *  @sa include/numkong/dots.h
+ *
+ *  Uses FMLAL (FEAT_FHM) for widening fp16->f32 multiply-accumulate, which is 20-48% faster
+ *  than the convert-then-FMA approach used in neonhalf.h.
+ */
+#ifndef NK_DOTS_NEONFHM_H
+#define NK_DOTS_NEONFHM_H
+
+#if NK_TARGET_ARM_
+#if NK_TARGET_NEONFHM
+
+#include "numkong/dot/neonfhm.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16+fp16fml"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=armv8.2-a+simd+fp16+fp16fml")
+#endif
+
+/* F16 GEMM using FMLAL: depth_simd_dimensions=8 (8 f16s = 16 bytes = NEON register width) */
+nk_define_cross_pack_size_(dots, f16, neonfhm, f16, f16, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/8,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, f16, neonfhm, f16, f16, nk_assign_from_to_, /*norm_value_type=*/f32,
+                      nk_dots_reduce_sumsq_f16_, /*depth_simd_dimensions=*/8,
+                      /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, f16, neonfhm, f16, f32, nk_b128_vec_t, nk_dot_f16x8_state_neonfhm_t, nk_b128_vec_t,
+                           nk_dot_f16x8_init_neonfhm, nk_load_b128_neon_, nk_partial_load_b16x8_serial_,
+                           nk_dot_f16x8_update_neonfhm, nk_dot_f16x8_finalize_neonfhm, nk_store_b128_neon_,
+                           nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, f16, neonfhm, f16, f16, f32, nk_b128_vec_t, nk_dot_f16x8_state_neonfhm_t, nk_b128_vec_t,
+                        nk_dot_f16x8_init_neonfhm, nk_load_b128_neon_, nk_partial_load_b16x8_serial_,
+                        nk_load_b128_neon_, nk_partial_load_b16x8_serial_, nk_dot_f16x8_update_neonfhm,
+                        nk_dot_f16x8_finalize_neonfhm, nk_store_b128_neon_, nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/8, /*dimensions_per_value=*/1)
+
+/* E4M3 GEMM via FMLAL: depth_simd_dimensions=16 (16 e4m3s = 16 bytes) */
+nk_define_cross_pack_size_(dots, e4m3, neonfhm, e4m3, e4m3, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/16,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, e4m3, neonfhm, e4m3, e4m3, nk_assign_from_to_, /*norm_value_type=*/f32,
+                      nk_dots_reduce_sumsq_e4m3_, /*depth_simd_dimensions=*/16,
+                      /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, e4m3, neonfhm, e4m3, f32, nk_b128_vec_t, nk_dot_e4m3x16_state_neonfhm_t, nk_b128_vec_t,
+                           nk_dot_e4m3x16_init_neonfhm, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                           nk_dot_e4m3x16_update_neonfhm, nk_dot_e4m3x16_finalize_neonfhm, nk_store_b128_neon_,
+                           nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, e4m3, neonfhm, e4m3, e4m3, f32, nk_b128_vec_t, nk_dot_e4m3x16_state_neonfhm_t,
+                        nk_b128_vec_t, nk_dot_e4m3x16_init_neonfhm, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                        nk_load_b128_neon_, nk_partial_load_b8x16_serial_, nk_dot_e4m3x16_update_neonfhm,
+                        nk_dot_e4m3x16_finalize_neonfhm, nk_store_b128_neon_, nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+
+/* E5M2 GEMM via FMLAL: depth_simd_dimensions=16 (16 e5m2s = 16 bytes) */
+nk_define_cross_pack_size_(dots, e5m2, neonfhm, e5m2, e5m2, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/16,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, e5m2, neonfhm, e5m2, e5m2, nk_assign_from_to_, /*norm_value_type=*/f32,
+                      nk_dots_reduce_sumsq_e5m2_, /*depth_simd_dimensions=*/16,
+                      /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, e5m2, neonfhm, e5m2, f32, nk_b128_vec_t, nk_dot_e5m2x16_state_neonfhm_t, nk_b128_vec_t,
+                           nk_dot_e5m2x16_init_neonfhm, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                           nk_dot_e5m2x16_update_neonfhm, nk_dot_e5m2x16_finalize_neonfhm, nk_store_b128_neon_,
+                           nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, e5m2, neonfhm, e5m2, e5m2, f32, nk_b128_vec_t, nk_dot_e5m2x16_state_neonfhm_t,
+                        nk_b128_vec_t, nk_dot_e5m2x16_init_neonfhm, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                        nk_load_b128_neon_, nk_partial_load_b8x16_serial_, nk_dot_e5m2x16_update_neonfhm,
+                        nk_dot_e5m2x16_finalize_neonfhm, nk_store_b128_neon_, nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_NEONFHM
+#endif // NK_TARGET_ARM_
+#endif // NK_DOTS_NEONFHM_H

package/include/numkong/dots/neonhalf.h

@@ -0,0 +1,57 @@
+/**
+ *  @brief SIMD-accelerated Batched Dot Products for NEON FP16.
+ *  @file include/numkong/dots/neonhalf.h
+ *  @author Ash Vardanian
+ *  @date December 27, 2025
+ *
+ *  @sa include/numkong/dots.h
+ */
+#ifndef NK_DOTS_NEONHALF_H
+#define NK_DOTS_NEONHALF_H
+
+#if NK_TARGET_ARM_
+#if NK_TARGET_NEONHALF
+
+#include "numkong/dot/neonhalf.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=armv8.2-a+simd+fp16")
+#endif
+
+/* F16 GEMM: depth_simd_dimensions=4 (4 f16s = 8 bytes = 64-bit input for direct f32 conversion) */
+nk_define_cross_pack_size_(dots, f16, neonhalf, f16, f16, /*norm_value_type=*/f32, /*depth_simd_dimensions=*/4,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, f16, neonhalf, f16, f16, nk_assign_from_to_, /*norm_value_type=*/f32,
+                      nk_dots_reduce_sumsq_f16_, /*depth_simd_dimensions=*/4,
+                      /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, f16, neonhalf, f16, f32, nk_b64_vec_t, nk_dot_f16x4_state_neonhalf_t, nk_b128_vec_t,
+                           nk_dot_f16x4_init_neonhalf, nk_load_b64_neon_, nk_partial_load_b16x4_serial_,
+                           nk_dot_f16x4_update_neonhalf, nk_dot_f16x4_finalize_neonhalf, nk_store_b128_neon_,
+                           nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/4, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, f16, neonhalf, f16, f16, f32, nk_b64_vec_t, nk_dot_f16x4_state_neonhalf_t, nk_b128_vec_t,
+                        nk_dot_f16x4_init_neonhalf, nk_load_b64_neon_, nk_partial_load_b16x4_serial_, nk_load_b64_neon_,
+                        nk_partial_load_b16x4_serial_, nk_dot_f16x4_update_neonhalf, nk_dot_f16x4_finalize_neonhalf,
+                        nk_store_b128_neon_, nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/4, /*dimensions_per_value=*/1)
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_NEONHALF
+#endif // NK_TARGET_ARM_
+#endif // NK_DOTS_NEONHALF_H

package/include/numkong/dots/neonsdot.h

@@ -0,0 +1,108 @@
+/**
+ *  @brief SIMD-accelerated Batched Dot Products for NEON SDOT.
+ *  @file include/numkong/dots/neonsdot.h
+ *  @author Ash Vardanian
+ *  @date December 27, 2025
+ *
+ *  @sa include/numkong/dots.h
+ */
+#ifndef NK_DOTS_NEONSDOT_H
+#define NK_DOTS_NEONSDOT_H
+
+#if NK_TARGET_ARM_
+#if NK_TARGET_NEONSDOT
+
+#include "numkong/dot/neonsdot.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+dotprod"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=armv8.2-a+dotprod")
+#endif
+
+/* I8 GEMM: depth_simd_dimensions=16 (16 i8s = 16 bytes = NEON register width) */
+nk_define_cross_pack_size_(dots, i8, neonsdot, i8, i8, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/16,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, i8, neonsdot, i8, i8, nk_assign_from_to_, /*norm_value_type=*/u32, nk_dots_reduce_sumsq_i8_,
+                      /*depth_simd_dimensions=*/16,
+                      /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, i8, neonsdot, i8, i32, nk_b128_vec_t, nk_dot_i8x16_state_neonsdot_t, nk_b128_vec_t,
+                           nk_dot_i8x16_init_neonsdot, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                           nk_dot_i8x16_update_neonsdot, nk_dot_i8x16_finalize_neonsdot, nk_store_b128_neon_,
+                           nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, i8, neonsdot, i8, i8, i32, nk_b128_vec_t, nk_dot_i8x16_state_neonsdot_t, nk_b128_vec_t,
+                        nk_dot_i8x16_init_neonsdot, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                        nk_load_b128_neon_, nk_partial_load_b8x16_serial_, nk_dot_i8x16_update_neonsdot,
+                        nk_dot_i8x16_finalize_neonsdot, nk_store_b128_neon_, nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+
+/* U8 GEMM: depth_simd_dimensions=16 (16 u8s = 16 bytes = NEON register width) */
+nk_define_cross_pack_size_(dots, u8, neonsdot, u8, u8, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/16,
+                           /*dimensions_per_value=*/1)
+nk_define_cross_pack_(dots, u8, neonsdot, u8, u8, nk_assign_from_to_, /*norm_value_type=*/u32, nk_dots_reduce_sumsq_u8_,
+                      /*depth_simd_dimensions=*/16,
+                      /*dimensions_per_value=*/1)
+nk_define_cross_symmetric_(dots, u8, neonsdot, u8, u32, nk_b128_vec_t, nk_dot_u8x16_state_neonsdot_t, nk_b128_vec_t,
+                           nk_dot_u8x16_init_neonsdot, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                           nk_dot_u8x16_update_neonsdot, nk_dot_u8x16_finalize_neonsdot, nk_store_b128_neon_,
+                           nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+nk_define_cross_packed_(dots, u8, neonsdot, u8, u8, u32, nk_b128_vec_t, nk_dot_u8x16_state_neonsdot_t, nk_b128_vec_t,
+                        nk_dot_u8x16_init_neonsdot, nk_load_b128_neon_, nk_partial_load_b8x16_serial_,
+                        nk_load_b128_neon_, nk_partial_load_b8x16_serial_, nk_dot_u8x16_update_neonsdot,
+                        nk_dot_u8x16_finalize_neonsdot, nk_store_b128_neon_, nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/16, /*dimensions_per_value=*/1)
+
+/* I4 GEMM: depth_simd_dimensions=32 (32 nibbles = 16 bytes = NEON register width) */
+nk_define_cross_pack_size_(dots, i4, neonsdot, i4x2, i4x2, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/32,
+                           /*dimensions_per_value=*/2)
+nk_define_cross_pack_(dots, i4, neonsdot, i4x2, i4x2, nk_assign_from_to_, /*norm_value_type=*/u32,
+                      nk_dots_reduce_sumsq_i4_, /*depth_simd_dimensions=*/32,
+                      /*dimensions_per_value=*/2)
+nk_define_cross_symmetric_(dots, i4, neonsdot, i4x2, i32, nk_b128_vec_t, nk_dot_i4x32_state_neonsdot_t, nk_b128_vec_t,
+                           nk_dot_i4x32_init_neonsdot, nk_load_b128_neon_, nk_partial_load_b4x32_serial_,
+                           nk_dot_i4x32_update_neonsdot, nk_dot_i4x32_finalize_neonsdot, nk_store_b128_neon_,
+                           nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/2)
+nk_define_cross_packed_(dots, i4, neonsdot, i4x2, i4x2, i32, nk_b128_vec_t, nk_dot_i4x32_state_neonsdot_t,
+                        nk_b128_vec_t, nk_dot_i4x32_init_neonsdot, nk_load_b128_neon_, nk_partial_load_b4x32_serial_,
+                        nk_load_b128_neon_, nk_partial_load_b4x32_serial_, nk_dot_i4x32_update_neonsdot,
+                        nk_dot_i4x32_finalize_neonsdot, nk_store_b128_neon_, nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/2)
+
+/* U4 GEMM: depth_simd_dimensions=32 (32 nibbles = 16 bytes = NEON register width) */
+nk_define_cross_pack_size_(dots, u4, neonsdot, u4x2, u4x2, /*norm_value_type=*/u32, /*depth_simd_dimensions=*/32,
+                           /*dimensions_per_value=*/2)
+nk_define_cross_pack_(dots, u4, neonsdot, u4x2, u4x2, nk_assign_from_to_, /*norm_value_type=*/u32,
+                      nk_dots_reduce_sumsq_u4_, /*depth_simd_dimensions=*/32,
+                      /*dimensions_per_value=*/2)
+nk_define_cross_symmetric_(dots, u4, neonsdot, u4x2, u32, nk_b128_vec_t, nk_dot_u4x32_state_neonsdot_t, nk_b128_vec_t,
+                           nk_dot_u4x32_init_neonsdot, nk_load_b128_neon_, nk_partial_load_b4x32_serial_,
+                           nk_dot_u4x32_update_neonsdot, nk_dot_u4x32_finalize_neonsdot, nk_store_b128_neon_,
+                           nk_partial_store_b32x4_serial_,
+                           /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/2)
+nk_define_cross_packed_(dots, u4, neonsdot, u4x2, u4x2, u32, nk_b128_vec_t, nk_dot_u4x32_state_neonsdot_t,
+                        nk_b128_vec_t, nk_dot_u4x32_init_neonsdot, nk_load_b128_neon_, nk_partial_load_b4x32_serial_,
+                        nk_load_b128_neon_, nk_partial_load_b4x32_serial_, nk_dot_u4x32_update_neonsdot,
+                        nk_dot_u4x32_finalize_neonsdot, nk_store_b128_neon_, nk_partial_store_b32x4_serial_,
+                        /*depth_simd_dimensions=*/32, /*dimensions_per_value=*/2)
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_NEONSDOT
+#endif // NK_TARGET_ARM_
+#endif // NK_DOTS_NEONSDOT_H