numkong 7.0.0

package/include/numkong/dot/sve.h

@@ -0,0 +1,379 @@
+/**
+ *  @brief SIMD-accelerated Dot Products for SVE.
+ *  @file include/numkong/dot/sve.h
+ *  @author Ash Vardanian
+ *  @date December 27, 2025
+ *
+ *  @sa include/numkong/dot.h
+ *
+ *  @section dot_sve_instructions ARM SVE Instructions
+ *
+ *      Intrinsic                   Instruction                     Latency     Throughput
+ *      svld1_f32                   LD1W (Z.S, P/Z, [Xn])           4-6cy       2/cy
+ *      svld2_f32                   LD2W (Z.S, P/Z, [Xn])           6-8cy       1/cy
+ *      svmla_f32_x                 FMLA (Z.S, P/M, Z.S, Z.S)       4cy         2/cy
+ *      svmls_f32_x                 FMLS (Z.S, P/M, Z.S, Z.S)       4cy         2/cy
+ *      svaddv_f32                  FADDV (S, P, Z.S)               6cy         1/cy
+ *      svdup_f32                   DUP (Z.S, #imm)                 1cy         2/cy
+ *      svwhilelt_b32               WHILELT (P.S, Xn, Xm)           2cy         1/cy
+ *      svptrue_b32                 PTRUE (P.S, pattern)            1cy         2/cy
+ *      svcntw                      CNTW (Xd)                       1cy         2/cy
+ *      svcntd                      CNTD (Xd)                       1cy         2/cy
+ *      svld1_f64                   LD1D (Z.D, P/Z, [Xn])           4-6cy       2/cy
+ *      svld2_f64                   LD2D (Z.D, P/Z, [Xn])           6-8cy       1/cy
+ *      svmla_f64_x                 FMLA (Z.D, P/M, Z.D, Z.D)       4cy         2/cy
+ *      svmls_f64_x                 FMLS (Z.D, P/M, Z.D, Z.D)       4cy         2/cy
+ *      svaddv_f64                  FADDV (D, P, Z.D)               6cy         1/cy
+ *
+ *  SVE vector widths vary across implementations: Graviton3 uses 256-bit, while Graviton4/5
+ *  and Apple M4+ use 128-bit. Code using svcntb() adapts automatically, but wider vectors
+ *  process more elements per iteration with identical latencies.
+ *
+ *  The FADDV horizontal reduction has higher latency (6cy) compared to vertical operations,
+ *  making it beneficial to accumulate in vector registers and reduce only at the end.
+ */
+#ifndef NK_DOT_SVE_H
+#define NK_DOT_SVE_H
+
+#if NK_TARGET_ARM_
+#if NK_TARGET_SVE
+
+#include "numkong/types.h"      // `nk_f32_t`
+#include "numkong/dot/serial.h" // `nk_u1x8_popcount_`
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=armv8.2-a+sve")
+#endif
+
+/** @brief Compensated horizontal sum of SVE f64 lanes via TwoSum tree reduction.
+ *
+ *  Uses svtbl to extract the upper half at each tree level. Out-of-range indices
+ *  return 0 (SVE spec), which is harmless since only the lower half is meaningful
+ *  after each halving stage.
+ */
+NK_INTERNAL nk_f64_t nk_dot_stable_sum_f64_sve_(svbool_t predicate, svfloat64_t sum, svfloat64_t compensation) {
+    // Stage 0: TwoSum merge of sum + compensation (parallel across all active lanes)
+    svfloat64_t tentative_sum_f64x = svadd_f64_x(predicate, sum, compensation);
+    svfloat64_t virtual_addend_f64x = svsub_f64_x(predicate, tentative_sum_f64x, sum);
+    svfloat64_t accumulated_error_f64x = svadd_f64_x(
+        predicate, svsub_f64_x(predicate, sum, svsub_f64_x(predicate, tentative_sum_f64x, virtual_addend_f64x)),
+        svsub_f64_x(predicate, compensation, virtual_addend_f64x));
+
+    // Tree reduction: TwoSum halving at each level, log2(VL) iterations
+    for (unsigned int half = (unsigned int)svcntd() / 2; half > 0; half >>= 1) {
+        svuint64_t upper_indices_u64x = svadd_n_u64_x(predicate, svindex_u64(0, 1), half);
+        svfloat64_t upper_sum_f64x = svtbl_f64(tentative_sum_f64x, upper_indices_u64x);
+        svfloat64_t upper_error_f64x = svtbl_f64(accumulated_error_f64x, upper_indices_u64x);
+        // TwoSum: lower_half + upper_half
+        svfloat64_t halved_tentative_sum_f64x = svadd_f64_x(predicate, tentative_sum_f64x, upper_sum_f64x);
+        svfloat64_t halved_virtual_addend_f64x = svsub_f64_x(predicate, halved_tentative_sum_f64x, tentative_sum_f64x);
+        svfloat64_t rounding_error_f64x = svadd_f64_x(
+            predicate,
+            svsub_f64_x(predicate, tentative_sum_f64x,
+                        svsub_f64_x(predicate, halved_tentative_sum_f64x, halved_virtual_addend_f64x)),
+            svsub_f64_x(predicate, upper_sum_f64x, halved_virtual_addend_f64x));
+        tentative_sum_f64x = halved_tentative_sum_f64x;
+        accumulated_error_f64x = svadd_f64_x(
+            predicate, svadd_f64_x(predicate, accumulated_error_f64x, upper_error_f64x), rounding_error_f64x);
+    }
+    // Result is in lane 0
+    svbool_t predicate_first_f64x = svwhilelt_b64_u64(0u, 1);
+    return svlastb_f64(predicate_first_f64x, tentative_sum_f64x) +
+           svlastb_f64(predicate_first_f64x, accumulated_error_f64x);
+}
+
+NK_PUBLIC void nk_dot_f32_sve(nk_f32_t const *a_scalars, nk_f32_t const *b_scalars, nk_size_t count_scalars,
+                              nk_f64_t *result) {
+    nk_size_t idx_scalars = 0;
+    nk_size_t const vector_length = svcntd();
+    svfloat64_t ab_f64x = svdup_f64(0.);
+    for (; idx_scalars < count_scalars; idx_scalars += vector_length) {
+        svbool_t predicate_f64x = svwhilelt_b64_u64(idx_scalars, count_scalars);
+        svfloat64_t a_f64x = svcvt_f64_f32_x(
+            predicate_f64x, svld1_f32(svwhilelt_b32_u64(idx_scalars, count_scalars), a_scalars + idx_scalars));
+        svfloat64_t b_f64x = svcvt_f64_f32_x(
+            predicate_f64x, svld1_f32(svwhilelt_b32_u64(idx_scalars, count_scalars), b_scalars + idx_scalars));
+        ab_f64x = svmla_f64_x(predicate_f64x, ab_f64x, a_f64x, b_f64x);
+    }
+    *result = svaddv_f64(svptrue_b64(), ab_f64x);
+}
+
+NK_PUBLIC void nk_dot_f32c_sve(nk_f32c_t const *a_pairs, nk_f32c_t const *b_pairs, nk_size_t count_pairs,
+                               nk_f64c_t *results) {
+    nk_size_t idx_pairs = 0;
+    nk_size_t const vector_length = svcntd();
+    svfloat64_t ab_real_f64x = svdup_f64(0.);
+    svfloat64_t ab_imag_f64x = svdup_f64(0.);
+    for (; idx_pairs < count_pairs; idx_pairs += vector_length) {
+        svbool_t predicate_f64x = svwhilelt_b64_u64(idx_pairs, count_pairs);
+        svbool_t predicate_f32x = svwhilelt_b32_u64(idx_pairs, count_pairs);
+        svfloat32x2_t a_f32x2 = svld2_f32(predicate_f32x, (nk_f32_t const *)(a_pairs + idx_pairs));
+        svfloat32x2_t b_f32x2 = svld2_f32(predicate_f32x, (nk_f32_t const *)(b_pairs + idx_pairs));
+        svfloat64_t a_real_f64x = svcvt_f64_f32_x(predicate_f64x, svget2_f32(a_f32x2, 0));
+        svfloat64_t a_imag_f64x = svcvt_f64_f32_x(predicate_f64x, svget2_f32(a_f32x2, 1));
+        svfloat64_t b_real_f64x = svcvt_f64_f32_x(predicate_f64x, svget2_f32(b_f32x2, 0));
+        svfloat64_t b_imag_f64x = svcvt_f64_f32_x(predicate_f64x, svget2_f32(b_f32x2, 1));
+        ab_real_f64x = svmla_f64_x(predicate_f64x, ab_real_f64x, a_real_f64x, b_real_f64x);
+        ab_real_f64x = svmls_f64_x(predicate_f64x, ab_real_f64x, a_imag_f64x, b_imag_f64x);
+        ab_imag_f64x = svmla_f64_x(predicate_f64x, ab_imag_f64x, a_real_f64x, b_imag_f64x);
+        ab_imag_f64x = svmla_f64_x(predicate_f64x, ab_imag_f64x, a_imag_f64x, b_real_f64x);
+    }
+    results->real = svaddv_f64(svptrue_b64(), ab_real_f64x);
+    results->imag = svaddv_f64(svptrue_b64(), ab_imag_f64x);
+}
+
+NK_PUBLIC void nk_vdot_f32c_sve(nk_f32c_t const *a_pairs, nk_f32c_t const *b_pairs, nk_size_t count_pairs,
+                                nk_f64c_t *results) {
+    nk_size_t idx_pairs = 0;
+    nk_size_t const vector_length = svcntd();
+    svfloat64_t ab_real_f64x = svdup_f64(0.);
+    svfloat64_t ab_imag_f64x = svdup_f64(0.);
+    for (; idx_pairs < count_pairs; idx_pairs += vector_length) {
+        svbool_t predicate_f64x = svwhilelt_b64_u64(idx_pairs, count_pairs);
+        svbool_t predicate_f32x = svwhilelt_b32_u64(idx_pairs, count_pairs);
+        svfloat32x2_t a_f32x2 = svld2_f32(predicate_f32x, (nk_f32_t const *)(a_pairs + idx_pairs));
+        svfloat32x2_t b_f32x2 = svld2_f32(predicate_f32x, (nk_f32_t const *)(b_pairs + idx_pairs));
+        svfloat64_t a_real_f64x = svcvt_f64_f32_x(predicate_f64x, svget2_f32(a_f32x2, 0));
+        svfloat64_t a_imag_f64x = svcvt_f64_f32_x(predicate_f64x, svget2_f32(a_f32x2, 1));
+        svfloat64_t b_real_f64x = svcvt_f64_f32_x(predicate_f64x, svget2_f32(b_f32x2, 0));
+        svfloat64_t b_imag_f64x = svcvt_f64_f32_x(predicate_f64x, svget2_f32(b_f32x2, 1));
+        ab_real_f64x = svmla_f64_x(predicate_f64x, ab_real_f64x, a_real_f64x, b_real_f64x);
+        ab_real_f64x = svmla_f64_x(predicate_f64x, ab_real_f64x, a_imag_f64x, b_imag_f64x);
+        ab_imag_f64x = svmla_f64_x(predicate_f64x, ab_imag_f64x, a_real_f64x, b_imag_f64x);
+        ab_imag_f64x = svmls_f64_x(predicate_f64x, ab_imag_f64x, a_imag_f64x, b_real_f64x);
+    }
+    results->real = svaddv_f64(svptrue_b64(), ab_real_f64x);
+    results->imag = svaddv_f64(svptrue_b64(), ab_imag_f64x);
+}
+
+NK_PUBLIC void nk_dot_f64_sve(nk_f64_t const *a_scalars, nk_f64_t const *b_scalars, nk_size_t count_scalars,
+                              nk_f64_t *result) {
+    // Dot2 (Ogita-Rump-Oishi) compensated accumulation via TwoProd + TwoSum
+    nk_size_t idx_scalars = 0;
+    svfloat64_t sum_f64x = svdup_f64(0.);
+    svfloat64_t compensation_f64x = svdup_f64(0.);
+    do {
+        svbool_t predicate_f64x = svwhilelt_b64_u64(idx_scalars, count_scalars);
+        svfloat64_t a_f64x = svld1_f64(predicate_f64x, a_scalars + idx_scalars);
+        svfloat64_t b_f64x = svld1_f64(predicate_f64x, b_scalars + idx_scalars);
+        // TwoProd: product = a*b, error = -(product - a*b) negated
+        svfloat64_t product_f64x = svmul_f64_x(predicate_f64x, a_f64x, b_f64x);
+        svfloat64_t product_error_f64x = svneg_f64_x(predicate_f64x,
+                                                     svnmls_f64_x(predicate_f64x, product_f64x, a_f64x, b_f64x));
+        // TwoSum: tentative_sum = sum + product
+        svfloat64_t tentative_sum_f64x = svadd_f64_x(predicate_f64x, sum_f64x, product_f64x);
+        svfloat64_t virtual_addend_f64x = svsub_f64_x(predicate_f64x, tentative_sum_f64x, sum_f64x);
+        svfloat64_t sum_error_f64x = svadd_f64_x(
+            predicate_f64x,
+            svsub_f64_x(predicate_f64x, sum_f64x, svsub_f64_x(predicate_f64x, tentative_sum_f64x, virtual_addend_f64x)),
+            svsub_f64_x(predicate_f64x, product_f64x, virtual_addend_f64x));
+        sum_f64x = tentative_sum_f64x;
+        compensation_f64x = svadd_f64_x(predicate_f64x, compensation_f64x,
+                                        svadd_f64_x(predicate_f64x, sum_error_f64x, product_error_f64x));
+        idx_scalars += svcntd();
+    } while (idx_scalars < count_scalars);
+    *result = nk_dot_stable_sum_f64_sve_(svptrue_b64(), sum_f64x, compensation_f64x);
+}
+
+NK_PUBLIC void nk_dot_f64c_sve(nk_f64c_t const *a_pairs, nk_f64c_t const *b_pairs, nk_size_t count_pairs,
+                               nk_f64c_t *results) {
+    // Dot2 compensated accumulation for complex dot product: (a_real + i*a_imag)(b_real + i*b_imag)
+    // real = a_real*b_real - a_imag*b_imag, imag = a_real*b_imag + a_imag*b_real
+    nk_size_t idx_pairs = 0;
+    svfloat64_t sum_real_f64x = svdup_f64(0.);
+    svfloat64_t comp_real_f64x = svdup_f64(0.);
+    svfloat64_t sum_imag_f64x = svdup_f64(0.);
+    svfloat64_t comp_imag_f64x = svdup_f64(0.);
+    do {
+        svbool_t predicate_f64x = svwhilelt_b64_u64(idx_pairs, count_pairs);
+        svfloat64x2_t a_f64x2 = svld2_f64(predicate_f64x, (nk_f64_t const *)(a_pairs + idx_pairs));
+        svfloat64x2_t b_f64x2 = svld2_f64(predicate_f64x, (nk_f64_t const *)(b_pairs + idx_pairs));
+        svfloat64_t a_real_f64x = svget2_f64(a_f64x2, 0);
+        svfloat64_t a_imag_f64x = svget2_f64(a_f64x2, 1);
+        svfloat64_t b_real_f64x = svget2_f64(b_f64x2, 0);
+        svfloat64_t b_imag_f64x = svget2_f64(b_f64x2, 1);
+
+        // TwoProd + TwoSum for real part: sum_real += a_real*b_real
+        {
+            svfloat64_t product_f64x = svmul_f64_x(predicate_f64x, a_real_f64x, b_real_f64x);
+            svfloat64_t product_error_f64x = svneg_f64_x(
+                predicate_f64x, svnmls_f64_x(predicate_f64x, product_f64x, a_real_f64x, b_real_f64x));
+            svfloat64_t tentative_sum_f64x = svadd_f64_x(predicate_f64x, sum_real_f64x, product_f64x);
+            svfloat64_t virtual_addend_f64x = svsub_f64_x(predicate_f64x, tentative_sum_f64x, sum_real_f64x);
+            svfloat64_t sum_error_f64x = svadd_f64_x(
+                predicate_f64x,
+                svsub_f64_x(predicate_f64x, sum_real_f64x,
+                            svsub_f64_x(predicate_f64x, tentative_sum_f64x, virtual_addend_f64x)),
+                svsub_f64_x(predicate_f64x, product_f64x, virtual_addend_f64x));
+            sum_real_f64x = tentative_sum_f64x;
+            comp_real_f64x = svadd_f64_x(predicate_f64x, comp_real_f64x,
+                                         svadd_f64_x(predicate_f64x, sum_error_f64x, product_error_f64x));
+        }
+        // TwoProd + TwoSum for real part: sum_real -= a_imag*b_imag
+        {
+            svfloat64_t product_f64x = svmul_f64_x(predicate_f64x, a_imag_f64x, b_imag_f64x);
+            svfloat64_t product_error_f64x = svneg_f64_x(
+                predicate_f64x, svnmls_f64_x(predicate_f64x, product_f64x, a_imag_f64x, b_imag_f64x));
+            svfloat64_t neg_product_f64x = svneg_f64_x(predicate_f64x, product_f64x);
+            svfloat64_t neg_product_error_f64x = svneg_f64_x(predicate_f64x, product_error_f64x);
+            svfloat64_t tentative_sum_f64x = svadd_f64_x(predicate_f64x, sum_real_f64x, neg_product_f64x);
+            svfloat64_t virtual_addend_f64x = svsub_f64_x(predicate_f64x, tentative_sum_f64x, sum_real_f64x);
+            svfloat64_t sum_error_f64x = svadd_f64_x(
+                predicate_f64x,
+                svsub_f64_x(predicate_f64x, sum_real_f64x,
+                            svsub_f64_x(predicate_f64x, tentative_sum_f64x, virtual_addend_f64x)),
+                svsub_f64_x(predicate_f64x, neg_product_f64x, virtual_addend_f64x));
+            sum_real_f64x = tentative_sum_f64x;
+            comp_real_f64x = svadd_f64_x(predicate_f64x, comp_real_f64x,
+                                         svadd_f64_x(predicate_f64x, sum_error_f64x, neg_product_error_f64x));
+        }
+        // TwoProd + TwoSum for imaginary part: sum_imag += a_real*b_imag
+        {
+            svfloat64_t product_f64x = svmul_f64_x(predicate_f64x, a_real_f64x, b_imag_f64x);
+            svfloat64_t product_error_f64x = svneg_f64_x(
+                predicate_f64x, svnmls_f64_x(predicate_f64x, product_f64x, a_real_f64x, b_imag_f64x));
+            svfloat64_t tentative_sum_f64x = svadd_f64_x(predicate_f64x, sum_imag_f64x, product_f64x);
+            svfloat64_t virtual_addend_f64x = svsub_f64_x(predicate_f64x, tentative_sum_f64x, sum_imag_f64x);
+            svfloat64_t sum_error_f64x = svadd_f64_x(
+                predicate_f64x,
+                svsub_f64_x(predicate_f64x, sum_imag_f64x,
+                            svsub_f64_x(predicate_f64x, tentative_sum_f64x, virtual_addend_f64x)),
+                svsub_f64_x(predicate_f64x, product_f64x, virtual_addend_f64x));
+            sum_imag_f64x = tentative_sum_f64x;
+            comp_imag_f64x = svadd_f64_x(predicate_f64x, comp_imag_f64x,
+                                         svadd_f64_x(predicate_f64x, sum_error_f64x, product_error_f64x));
+        }
+        // TwoProd + TwoSum for imaginary part: sum_imag += a_imag*b_real
+        {
+            svfloat64_t product_f64x = svmul_f64_x(predicate_f64x, a_imag_f64x, b_real_f64x);
+            svfloat64_t product_error_f64x = svneg_f64_x(
+                predicate_f64x, svnmls_f64_x(predicate_f64x, product_f64x, a_imag_f64x, b_real_f64x));
+            svfloat64_t tentative_sum_f64x = svadd_f64_x(predicate_f64x, sum_imag_f64x, product_f64x);
+            svfloat64_t virtual_addend_f64x = svsub_f64_x(predicate_f64x, tentative_sum_f64x, sum_imag_f64x);
+            svfloat64_t sum_error_f64x = svadd_f64_x(
+                predicate_f64x,
+                svsub_f64_x(predicate_f64x, sum_imag_f64x,
+                            svsub_f64_x(predicate_f64x, tentative_sum_f64x, virtual_addend_f64x)),
+                svsub_f64_x(predicate_f64x, product_f64x, virtual_addend_f64x));
+            sum_imag_f64x = tentative_sum_f64x;
+            comp_imag_f64x = svadd_f64_x(predicate_f64x, comp_imag_f64x,
+                                         svadd_f64_x(predicate_f64x, sum_error_f64x, product_error_f64x));
+        }
+        idx_pairs += svcntd();
+    } while (idx_pairs < count_pairs);
+    svbool_t predicate_all_f64x = svptrue_b64();
+    results->real = nk_dot_stable_sum_f64_sve_(predicate_all_f64x, sum_real_f64x, comp_real_f64x);
+    results->imag = nk_dot_stable_sum_f64_sve_(predicate_all_f64x, sum_imag_f64x, comp_imag_f64x);
+}
+
+NK_PUBLIC void nk_vdot_f64c_sve(nk_f64c_t const *a_pairs, nk_f64c_t const *b_pairs, nk_size_t count_pairs,
+                                nk_f64c_t *results) {
+    // Dot2 compensated conjugate dot product: conj(a) · b = (a_real - i*a_imag)(b_real + i*b_imag)
+    // real = a_real*b_real + a_imag*b_imag, imag = a_real*b_imag - a_imag*b_real
+    nk_size_t idx_pairs = 0;
+    svfloat64_t sum_real_f64x = svdup_f64(0.);
+    svfloat64_t comp_real_f64x = svdup_f64(0.);
+    svfloat64_t sum_imag_f64x = svdup_f64(0.);
+    svfloat64_t comp_imag_f64x = svdup_f64(0.);
+    do {
+        svbool_t predicate_f64x = svwhilelt_b64_u64(idx_pairs, count_pairs);
+        svfloat64x2_t a_f64x2 = svld2_f64(predicate_f64x, (nk_f64_t const *)(a_pairs + idx_pairs));
+        svfloat64x2_t b_f64x2 = svld2_f64(predicate_f64x, (nk_f64_t const *)(b_pairs + idx_pairs));
+        svfloat64_t a_real_f64x = svget2_f64(a_f64x2, 0);
+        svfloat64_t a_imag_f64x = svget2_f64(a_f64x2, 1);
+        svfloat64_t b_real_f64x = svget2_f64(b_f64x2, 0);
+        svfloat64_t b_imag_f64x = svget2_f64(b_f64x2, 1);
+
+        // TwoProd + TwoSum for real part: sum_real += a_real*b_real
+        {
+            svfloat64_t product_f64x = svmul_f64_x(predicate_f64x, a_real_f64x, b_real_f64x);
+            svfloat64_t product_error_f64x = svneg_f64_x(
+                predicate_f64x, svnmls_f64_x(predicate_f64x, product_f64x, a_real_f64x, b_real_f64x));
+            svfloat64_t tentative_sum_f64x = svadd_f64_x(predicate_f64x, sum_real_f64x, product_f64x);
+            svfloat64_t virtual_addend_f64x = svsub_f64_x(predicate_f64x, tentative_sum_f64x, sum_real_f64x);
+            svfloat64_t sum_error_f64x = svadd_f64_x(
+                predicate_f64x,
+                svsub_f64_x(predicate_f64x, sum_real_f64x,
+                            svsub_f64_x(predicate_f64x, tentative_sum_f64x, virtual_addend_f64x)),
+                svsub_f64_x(predicate_f64x, product_f64x, virtual_addend_f64x));
+            sum_real_f64x = tentative_sum_f64x;
+            comp_real_f64x = svadd_f64_x(predicate_f64x, comp_real_f64x,
+                                         svadd_f64_x(predicate_f64x, sum_error_f64x, product_error_f64x));
+        }
+        // TwoProd + TwoSum for real part: sum_real += a_imag*b_imag (conjugate: + not -)
+        {
+            svfloat64_t product_f64x = svmul_f64_x(predicate_f64x, a_imag_f64x, b_imag_f64x);
+            svfloat64_t product_error_f64x = svneg_f64_x(
+                predicate_f64x, svnmls_f64_x(predicate_f64x, product_f64x, a_imag_f64x, b_imag_f64x));
+            svfloat64_t tentative_sum_f64x = svadd_f64_x(predicate_f64x, sum_real_f64x, product_f64x);
+            svfloat64_t virtual_addend_f64x = svsub_f64_x(predicate_f64x, tentative_sum_f64x, sum_real_f64x);
+            svfloat64_t sum_error_f64x = svadd_f64_x(
+                predicate_f64x,
+                svsub_f64_x(predicate_f64x, sum_real_f64x,
+                            svsub_f64_x(predicate_f64x, tentative_sum_f64x, virtual_addend_f64x)),
+                svsub_f64_x(predicate_f64x, product_f64x, virtual_addend_f64x));
+            sum_real_f64x = tentative_sum_f64x;
+            comp_real_f64x = svadd_f64_x(predicate_f64x, comp_real_f64x,
+                                         svadd_f64_x(predicate_f64x, sum_error_f64x, product_error_f64x));
+        }
+        // TwoProd + TwoSum for imaginary part: sum_imag += a_real*b_imag
+        {
+            svfloat64_t product_f64x = svmul_f64_x(predicate_f64x, a_real_f64x, b_imag_f64x);
+            svfloat64_t product_error_f64x = svneg_f64_x(
+                predicate_f64x, svnmls_f64_x(predicate_f64x, product_f64x, a_real_f64x, b_imag_f64x));
+            svfloat64_t tentative_sum_f64x = svadd_f64_x(predicate_f64x, sum_imag_f64x, product_f64x);
+            svfloat64_t virtual_addend_f64x = svsub_f64_x(predicate_f64x, tentative_sum_f64x, sum_imag_f64x);
+            svfloat64_t sum_error_f64x = svadd_f64_x(
+                predicate_f64x,
+                svsub_f64_x(predicate_f64x, sum_imag_f64x,
+                            svsub_f64_x(predicate_f64x, tentative_sum_f64x, virtual_addend_f64x)),
+                svsub_f64_x(predicate_f64x, product_f64x, virtual_addend_f64x));
+            sum_imag_f64x = tentative_sum_f64x;
+            comp_imag_f64x = svadd_f64_x(predicate_f64x, comp_imag_f64x,
+                                         svadd_f64_x(predicate_f64x, sum_error_f64x, product_error_f64x));
+        }
+        // TwoProd + TwoSum for imaginary part: sum_imag -= a_imag*b_real (conjugate: - not +)
+        {
+            svfloat64_t product_f64x = svmul_f64_x(predicate_f64x, a_imag_f64x, b_real_f64x);
+            svfloat64_t product_error_f64x = svneg_f64_x(
+                predicate_f64x, svnmls_f64_x(predicate_f64x, product_f64x, a_imag_f64x, b_real_f64x));
+            svfloat64_t neg_product_f64x = svneg_f64_x(predicate_f64x, product_f64x);
+            svfloat64_t neg_product_error_f64x = svneg_f64_x(predicate_f64x, product_error_f64x);
+            svfloat64_t tentative_sum_f64x = svadd_f64_x(predicate_f64x, sum_imag_f64x, neg_product_f64x);
+            svfloat64_t virtual_addend_f64x = svsub_f64_x(predicate_f64x, tentative_sum_f64x, sum_imag_f64x);
+            svfloat64_t sum_error_f64x = svadd_f64_x(
+                predicate_f64x,
+                svsub_f64_x(predicate_f64x, sum_imag_f64x,
+                            svsub_f64_x(predicate_f64x, tentative_sum_f64x, virtual_addend_f64x)),
+                svsub_f64_x(predicate_f64x, neg_product_f64x, virtual_addend_f64x));
+            sum_imag_f64x = tentative_sum_f64x;
+            comp_imag_f64x = svadd_f64_x(predicate_f64x, comp_imag_f64x,
+                                         svadd_f64_x(predicate_f64x, sum_error_f64x, neg_product_error_f64x));
+        }
+        idx_pairs += svcntd();
+    } while (idx_pairs < count_pairs);
+    svbool_t predicate_all_f64x = svptrue_b64();
+    results->real = nk_dot_stable_sum_f64_sve_(predicate_all_f64x, sum_real_f64x, comp_real_f64x);
+    results->imag = nk_dot_stable_sum_f64_sve_(predicate_all_f64x, sum_imag_f64x, comp_imag_f64x);
+}
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_SVE
+#endif // NK_TARGET_ARM_
+#endif // NK_DOT_SVE_H

package/include/numkong/dot/svebfdot.h

@@ -0,0 +1,74 @@
+/**
+ *  @brief SIMD-accelerated Dot Products for SVE BF16.
+ *  @file include/numkong/dot/svebfdot.h
+ *  @author Ash Vardanian
+ *  @date March 16, 2026
+ *
+ *  @sa include/numkong/dot.h
+ *
+ *  @section dot_svebfdot_instructions ARM SVE+BF16 Instructions
+ *
+ *      Intrinsic                   Instruction                     Latency     Throughput
+ *      svld1_bf16                  LD1H (Z.H, P/Z, [Xn])           4-6cy       2/cy
+ *      svbfdot_f32                 BFDOT (Z.S, Z.H, Z.H)           4cy         2/cy
+ *      svaddv_f32                  FADDV (S, P, Z.S)               6cy         1/cy
+ *      svdup_f32                   DUP (Z.S, #imm)                 1cy         2/cy
+ *      svwhilelt_b16               WHILELT (P.H, Xn, Xm)           2cy         1/cy
+ *      svcnth                      CNTH (Xd)                       1cy         2/cy
+ *
+ *  SVE vector widths vary across implementations: Graviton3 uses 256-bit, while Graviton4/5
+ *  and Apple M4+ use 128-bit. Code using svcnth() adapts automatically, but wider vectors
+ *  process more elements per iteration with identical latencies.
+ *
+ *  The BFDOT instruction fuses two BF16 multiplications with FP32 accumulation per lane,
+ *  providing 4x the throughput of convert-then-FMA sequences. Each BFDOT processes
+ *  pairs of BF16 values, accumulating directly into FP32 without explicit conversion.
+ */
+#ifndef NK_DOT_SVEBFDOT_H
+#define NK_DOT_SVEBFDOT_H
+
+#if NK_TARGET_ARM_
+#if NK_TARGET_SVEBFDOT
+
+#include "numkong/types.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+bf16"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=armv8.2-a+sve+bf16")
+#endif
+
+NK_PUBLIC void nk_dot_bf16_svebfdot(nk_bf16_t const *a_scalars, nk_bf16_t const *b_scalars, nk_size_t count_scalars,
+                                    nk_f32_t *result) {
+    nk_size_t idx_scalars = 0;
+    svfloat32_t sum_f32x = svdup_f32(0);
+    nk_bf16_for_arm_simd_t const *a = (nk_bf16_for_arm_simd_t const *)(a_scalars);
+    nk_bf16_for_arm_simd_t const *b = (nk_bf16_for_arm_simd_t const *)(b_scalars);
+    do {
+        svbool_t predicate_bf16x = svwhilelt_b16_u64(idx_scalars, count_scalars);
+        svbfloat16_t a_bf16x = svld1_bf16(predicate_bf16x, a + idx_scalars);
+        svbfloat16_t b_bf16x = svld1_bf16(predicate_bf16x, b + idx_scalars);
+        sum_f32x = svbfdot_f32(sum_f32x, a_bf16x, b_bf16x);
+        idx_scalars += svcnth();
+    } while (idx_scalars < count_scalars);
+    *result = svaddv_f32(svptrue_b32(), sum_f32x);
+}
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_SVEBFDOT
+#endif // NK_TARGET_ARM_
+#endif // NK_DOT_SVEBFDOT_H

package/include/numkong/dot/svehalf.h

@@ -0,0 +1,123 @@
+/**
+ *  @brief SIMD-accelerated Dot Products for SVE FP16.
+ *  @file include/numkong/dot/sve.h
+ *  @author Ash Vardanian
+ *  @date December 27, 2025
+ *
+ *  @sa include/numkong/dot.h
+ *
+ *  @section dot_svehalf_instructions ARM SVE+FP16 Instructions
+ *
+ *      Intrinsic                   Instruction                     Latency     Throughput
+ *      svld1_f16                   LD1H (Z.H, P/Z, [Xn])           4-6cy       2/cy
+ *      svld2_f16                   LD2H (Z.H, P/Z, [Xn])           6-8cy       1/cy
+ *      svmla_f16_x                 FMLA (Z.H, P/M, Z.H, Z.H)       4cy         2/cy
+ *      svmls_f16_x                 FMLS (Z.H, P/M, Z.H, Z.H)       4cy         2/cy
+ *      svaddv_f16                  FADDV (H, P, Z.H)               6cy         1/cy
+ *      svdup_f16                   DUP (Z.H, #imm)                 1cy         2/cy
+ *      svwhilelt_b16               WHILELT (P.H, Xn, Xm)           2cy         1/cy
+ *      svptrue_b16                 PTRUE (P.H, pattern)            1cy         2/cy
+ *      svcnth                      CNTH (Xd)                       1cy         2/cy
+ *
+ *  SVE vector widths vary across implementations: Graviton3 uses 256-bit, while Graviton4/5
+ *  and Apple M4+ use 128-bit. Code using svcntb() adapts automatically, but wider vectors
+ *  process more elements per iteration with identical latencies.
+ *
+ *  FP16 operations double the element count per vector compared to FP32, providing higher
+ *  throughput at the cost of reduced precision. The FADDV reduction remains the bottleneck.
+ */
+#ifndef NK_DOT_SVEHALF_H
+#define NK_DOT_SVEHALF_H
+
+#if NK_TARGET_ARM_
+#if NK_TARGET_SVEHALF
+
+#include "numkong/types.h"      // `nk_f16_t`
+#include "numkong/dot/serial.h" // `nk_u1x8_popcount_`
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+fp16"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=armv8.2-a+sve+fp16")
+#endif
+
+NK_PUBLIC void nk_dot_f16_svehalf(nk_f16_t const *a_scalars, nk_f16_t const *b_scalars, nk_size_t count_scalars,
+                                  nk_f32_t *result) {
+    nk_size_t idx_scalars = 0;
+    svfloat32_t ab_f32x = svdup_f32(0);
+    do {
+        svbool_t predicate_f32x = svwhilelt_b32_u64(idx_scalars, count_scalars);
+        svfloat16_t a_f16x = svld1_f16(predicate_f32x, (nk_f16_for_arm_simd_t const *)(a_scalars) + idx_scalars);
+        svfloat16_t b_f16x = svld1_f16(predicate_f32x, (nk_f16_for_arm_simd_t const *)(b_scalars) + idx_scalars);
+        svfloat32_t a_f32x = svcvt_f32_f16_x(predicate_f32x, a_f16x);
+        svfloat32_t b_f32x = svcvt_f32_f16_x(predicate_f32x, b_f16x);
+        ab_f32x = svmla_f32_x(predicate_f32x, ab_f32x, a_f32x, b_f32x);
+        idx_scalars += svcntw();
+    } while (idx_scalars < count_scalars);
+    *result = svaddv_f32(svptrue_b32(), ab_f32x);
+}
+
+NK_PUBLIC void nk_dot_f16c_svehalf(nk_f16c_t const *a_pairs, nk_f16c_t const *b_pairs, nk_size_t count_pairs,
+                                   nk_f32c_t *results) {
+    nk_size_t idx_scalars = 0;
+    svfloat32_t ab_real_f32x = svdup_f32(0);
+    svfloat32_t ab_imag_f32x = svdup_f32(0);
+    do {
+        svbool_t predicate_f32x = svwhilelt_b32_u64(idx_scalars, count_pairs);
+        svfloat16x2_t a_f16x2 = svld2_f16(predicate_f32x, (nk_f16_for_arm_simd_t const *)(a_pairs) + idx_scalars * 2);
+        svfloat16x2_t b_f16x2 = svld2_f16(predicate_f32x, (nk_f16_for_arm_simd_t const *)(b_pairs) + idx_scalars * 2);
+        svfloat32_t a_real_f32x = svcvt_f32_f16_x(predicate_f32x, svget2_f16(a_f16x2, 0));
+        svfloat32_t a_imag_f32x = svcvt_f32_f16_x(predicate_f32x, svget2_f16(a_f16x2, 1));
+        svfloat32_t b_real_f32x = svcvt_f32_f16_x(predicate_f32x, svget2_f16(b_f16x2, 0));
+        svfloat32_t b_imag_f32x = svcvt_f32_f16_x(predicate_f32x, svget2_f16(b_f16x2, 1));
+        ab_real_f32x = svmla_f32_x(predicate_f32x, ab_real_f32x, a_real_f32x, b_real_f32x);
+        ab_real_f32x = svmls_f32_x(predicate_f32x, ab_real_f32x, a_imag_f32x, b_imag_f32x);
+        ab_imag_f32x = svmla_f32_x(predicate_f32x, ab_imag_f32x, a_real_f32x, b_imag_f32x);
+        ab_imag_f32x = svmla_f32_x(predicate_f32x, ab_imag_f32x, a_imag_f32x, b_real_f32x);
+        idx_scalars += svcntw();
+    } while (idx_scalars < count_pairs);
+    results->real = svaddv_f32(svptrue_b32(), ab_real_f32x);
+    results->imag = svaddv_f32(svptrue_b32(), ab_imag_f32x);
+}
+
+NK_PUBLIC void nk_vdot_f16c_svehalf(nk_f16c_t const *a_pairs, nk_f16c_t const *b_pairs, nk_size_t count_pairs,
+                                    nk_f32c_t *results) {
+    nk_size_t idx_scalars = 0;
+    svfloat32_t ab_real_f32x = svdup_f32(0);
+    svfloat32_t ab_imag_f32x = svdup_f32(0);
+    do {
+        svbool_t predicate_f32x = svwhilelt_b32_u64(idx_scalars, count_pairs);
+        svfloat16x2_t a_f16x2 = svld2_f16(predicate_f32x, (nk_f16_for_arm_simd_t const *)(a_pairs) + idx_scalars * 2);
+        svfloat16x2_t b_f16x2 = svld2_f16(predicate_f32x, (nk_f16_for_arm_simd_t const *)(b_pairs) + idx_scalars * 2);
+        svfloat32_t a_real_f32x = svcvt_f32_f16_x(predicate_f32x, svget2_f16(a_f16x2, 0));
+        svfloat32_t a_imag_f32x = svcvt_f32_f16_x(predicate_f32x, svget2_f16(a_f16x2, 1));
+        svfloat32_t b_real_f32x = svcvt_f32_f16_x(predicate_f32x, svget2_f16(b_f16x2, 0));
+        svfloat32_t b_imag_f32x = svcvt_f32_f16_x(predicate_f32x, svget2_f16(b_f16x2, 1));
+        ab_real_f32x = svmla_f32_x(predicate_f32x, ab_real_f32x, a_real_f32x, b_real_f32x);
+        ab_real_f32x = svmla_f32_x(predicate_f32x, ab_real_f32x, a_imag_f32x, b_imag_f32x);
+        ab_imag_f32x = svmla_f32_x(predicate_f32x, ab_imag_f32x, a_real_f32x, b_imag_f32x);
+        ab_imag_f32x = svmls_f32_x(predicate_f32x, ab_imag_f32x, a_imag_f32x, b_real_f32x);
+        idx_scalars += svcntw();
+    } while (idx_scalars < count_pairs);
+    results->real = svaddv_f32(svptrue_b32(), ab_real_f32x);
+    results->imag = svaddv_f32(svptrue_b32(), ab_imag_f32x);
+}
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_SVEHALF
+#endif // NK_TARGET_ARM_
+#endif // NK_DOT_SVEHALF_H