numkong 7.0.0

package/include/numkong/curved/rvv.h

@@ -0,0 +1,305 @@
+/**
+ *  @brief SIMD-accelerated Curved Space Distances for RISC-V.
+ *  @file include/numkong/curved/rvv.h
+ *  @author Ash Vardanian
+ *  @date February 6, 2026
+ *
+ *  @sa include/numkong/curved.h
+ *
+ *  Implements bilinear forms and Mahalanobis distance using RVV 1.0:
+ *  - f32 inputs use f32 SIMD accumulation with vfredusum ordered reduction
+ *  - f64 inputs use f64 SIMD accumulation with vfredusum ordered reduction
+ *  - f16/bf16 inputs are converted to f32 via cast helpers, then accumulated in f32
+ *  - Complex bilinear forms delegate to serial implementations
+ */
+#ifndef NK_CURVED_RVV_H
+#define NK_CURVED_RVV_H
+
+#if NK_TARGET_RISCV_
+#if NK_TARGET_RVV
+
+#include "numkong/types.h"
+#include "numkong/curved/serial.h"
+#include "numkong/cast/rvv.h"
+#include "numkong/spatial/rvv.h" // `nk_f64_sqrt_rvv`
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=+v"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=+v")
+#endif
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+NK_PUBLIC void nk_bilinear_f32_rvv(nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t n,
+                                   nk_f64_t *result) {
+    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
+    nk_f64_t outer_sum = 0;
+    for (nk_size_t i = 0; i < n; ++i) {
+        vfloat64m4_t inner_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, vlmax);
+        nk_f32_t const *c_row = c + i * n;
+        nk_size_t remaining = n;
+        for (nk_size_t vector_length; remaining > 0; remaining -= vector_length, c_row += vector_length) {
+            vector_length = __riscv_vsetvl_e32m2(remaining);
+            vfloat32m2_t c_f32m2 = __riscv_vle32_v_f32m2(c_row, vector_length);
+            vfloat32m2_t b_f32m2 = __riscv_vle32_v_f32m2(b + (n - remaining), vector_length);
+            inner_f64m4 = __riscv_vfwmacc_vv_f64m4_tu(inner_f64m4, c_f32m2, b_f32m2, vector_length);
+        }
+        vfloat64m1_t zero_f64m1 = __riscv_vfmv_v_f_f64m1(0.0, 1);
+        nk_f64_t inner_val = __riscv_vfmv_f_s_f64m1_f64(
+            __riscv_vfredusum_vs_f64m4_f64m1(inner_f64m4, zero_f64m1, vlmax));
+        outer_sum += (nk_f64_t)a[i] * inner_val;
+    }
+    *result = outer_sum;
+}
+
+NK_PUBLIC void nk_bilinear_f64_rvv(nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t n,
+                                   nk_f64_t *result) {
+    nk_size_t vlmax = __riscv_vsetvlmax_e64m4();
+    vfloat64m1_t sum_f64m1 = __riscv_vfmv_v_f_f64m1(0.0, 1);
+    nk_f64_t outer_compensation = 0;
+    for (nk_size_t i = 0; i < n; ++i) {
+        vfloat64m4_t inner_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, vlmax);
+        vfloat64m4_t compensation_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, vlmax);
+        nk_f64_t const *c_row = c + i * n;
+        nk_size_t remaining = n;
+        for (nk_size_t vector_length; remaining > 0; remaining -= vector_length, c_row += vector_length) {
+            vector_length = __riscv_vsetvl_e64m4(remaining);
+            vfloat64m4_t vc_f64m4 = __riscv_vle64_v_f64m4(c_row, vector_length);
+            vfloat64m4_t vb_f64m4 = __riscv_vle64_v_f64m4(b + (n - remaining), vector_length);
+            vfloat64m4_t product_f64m4 = __riscv_vfmul_vv_f64m4(vc_f64m4, vb_f64m4, vector_length);
+            vfloat64m4_t corrected_term_f64m4 = __riscv_vfsub_vv_f64m4(product_f64m4, compensation_f64m4,
+                                                                       vector_length);
+            vfloat64m4_t running_sum_f64m4 = __riscv_vfadd_vv_f64m4_tu(inner_f64m4, inner_f64m4, corrected_term_f64m4,
+                                                                       vector_length);
+            compensation_f64m4 = __riscv_vfsub_vv_f64m4_tu(
+                compensation_f64m4, __riscv_vfsub_vv_f64m4(running_sum_f64m4, inner_f64m4, vector_length),
+                corrected_term_f64m4, vector_length);
+            inner_f64m4 = running_sum_f64m4;
+        }
+        vfloat64m1_t zero_f64m1 = __riscv_vfmv_v_f_f64m1(0.0, 1);
+        nk_f64_t inner_val = __riscv_vfmv_f_s_f64m1_f64(
+            __riscv_vfredusum_vs_f64m4_f64m1(inner_f64m4, zero_f64m1, vlmax));
+        nk_f64_t product_outer = a[i] * inner_val;
+        nk_f64_t old_sum = __riscv_vfmv_f_s_f64m1_f64(sum_f64m1);
+        nk_f64_t new_sum = old_sum + product_outer;
+        if (nk_f64_abs_(old_sum) >= nk_f64_abs_(product_outer))
+            outer_compensation += (old_sum - new_sum) + product_outer;
+        else outer_compensation += (product_outer - new_sum) + old_sum;
+        sum_f64m1 = __riscv_vfmv_v_f_f64m1(new_sum, 1);
+    }
+    *result = __riscv_vfmv_f_s_f64m1_f64(sum_f64m1) + outer_compensation;
+}
+
+NK_PUBLIC void nk_bilinear_f16_rvv(nk_f16_t const *a, nk_f16_t const *b, nk_f16_t const *c, nk_size_t n,
+                                   nk_f32_t *result) {
+    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
+    vfloat32m1_t sum_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
+    for (nk_size_t i = 0; i < n; ++i) {
+        // Convert a[i] from f16 to f32
+        nk_f32_t a_i;
+        nk_f16_to_f32_serial(a + i, &a_i);
+
+        vfloat32m2_t inner_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
+        nk_f16_t const *c_row = c + i * n;
+        nk_size_t remaining = n;
+        for (nk_size_t vector_length; remaining > 0; remaining -= vector_length, c_row += vector_length) {
+            vector_length = __riscv_vsetvl_e16m1(remaining);
+            // Load f16 as u16 bits and convert to f32
+            vuint16m1_t vc_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)c_row, vector_length);
+            vuint16m1_t vb_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)(b + (n - remaining)), vector_length);
+            vfloat32m2_t vc_f32m2 = nk_f16m1_to_f32m2_rvv_(vc_u16m1, vector_length);
+            vfloat32m2_t vb_f32m2 = nk_f16m1_to_f32m2_rvv_(vb_u16m1, vector_length);
+            inner_f32m2 = __riscv_vfmacc_vv_f32m2_tu(inner_f32m2, vc_f32m2, vb_f32m2, vector_length);
+        }
+        vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
+        nk_f32_t inner_val = __riscv_vfmv_f_s_f32m1_f32(
+            __riscv_vfredusum_vs_f32m2_f32m1(inner_f32m2, zero_f32m1, vlmax));
+        sum_f32m1 = __riscv_vfmv_v_f_f32m1(__riscv_vfmv_f_s_f32m1_f32(sum_f32m1) + a_i * inner_val, 1);
+    }
+    *result = __riscv_vfmv_f_s_f32m1_f32(sum_f32m1);
+}
+
+NK_PUBLIC void nk_bilinear_bf16_rvv(nk_bf16_t const *a, nk_bf16_t const *b, nk_bf16_t const *c, nk_size_t n,
+                                    nk_f32_t *result) {
+    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
+    vfloat32m1_t sum_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
+    for (nk_size_t i = 0; i < n; ++i) {
+        // Convert a[i] from bf16 to f32
+        nk_f32_t a_i;
+        nk_bf16_to_f32_serial(a + i, &a_i);
+
+        vfloat32m2_t inner_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
+        nk_bf16_t const *c_row = c + i * n;
+        nk_size_t remaining = n;
+        for (nk_size_t vector_length; remaining > 0; remaining -= vector_length, c_row += vector_length) {
+            vector_length = __riscv_vsetvl_e16m1(remaining);
+            // Load bf16 as u16 bits and convert to f32
+            vuint16m1_t vc_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)c_row, vector_length);
+            vuint16m1_t vb_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)(b + (n - remaining)), vector_length);
+            vfloat32m2_t vc_f32m2 = nk_bf16m1_to_f32m2_rvv_(vc_u16m1, vector_length);
+            vfloat32m2_t vb_f32m2 = nk_bf16m1_to_f32m2_rvv_(vb_u16m1, vector_length);
+            inner_f32m2 = __riscv_vfmacc_vv_f32m2_tu(inner_f32m2, vc_f32m2, vb_f32m2, vector_length);
+        }
+        vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
+        nk_f32_t inner_val = __riscv_vfmv_f_s_f32m1_f32(
+            __riscv_vfredusum_vs_f32m2_f32m1(inner_f32m2, zero_f32m1, vlmax));
+        sum_f32m1 = __riscv_vfmv_v_f_f32m1(__riscv_vfmv_f_s_f32m1_f32(sum_f32m1) + a_i * inner_val, 1);
+    }
+    *result = __riscv_vfmv_f_s_f32m1_f32(sum_f32m1);
+}
+
+NK_PUBLIC void nk_mahalanobis_f32_rvv(nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t n,
+                                      nk_f64_t *result) {
+    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
+    nk_f64_t outer_sum = 0;
+    for (nk_size_t i = 0; i < n; ++i) {
+        nk_f64_t diff_i = (nk_f64_t)a[i] - (nk_f64_t)b[i];
+        vfloat64m4_t inner_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, vlmax);
+        nk_f32_t const *c_row = c + i * n;
+        nk_size_t remaining = n;
+        for (nk_size_t vector_length; remaining > 0; remaining -= vector_length, c_row += vector_length) {
+            vector_length = __riscv_vsetvl_e32m2(remaining);
+            nk_size_t j = n - remaining;
+            vfloat32m2_t c_f32m2 = __riscv_vle32_v_f32m2(c_row, vector_length);
+            vfloat32m2_t a_f32m2 = __riscv_vle32_v_f32m2(a + j, vector_length);
+            vfloat32m2_t b_f32m2 = __riscv_vle32_v_f32m2(b + j, vector_length);
+            vfloat64m4_t diff_f64m4 = __riscv_vfwsub_vv_f64m4(a_f32m2, b_f32m2, vector_length);
+            vfloat64m4_t c_f64m4 = __riscv_vfwcvt_f_f_v_f64m4(c_f32m2, vector_length);
+            inner_f64m4 = __riscv_vfmacc_vv_f64m4_tu(inner_f64m4, c_f64m4, diff_f64m4, vector_length);
+        }
+        vfloat64m1_t zero_f64m1 = __riscv_vfmv_v_f_f64m1(0.0, 1);
+        nk_f64_t inner_val = __riscv_vfmv_f_s_f64m1_f64(
+            __riscv_vfredusum_vs_f64m4_f64m1(inner_f64m4, zero_f64m1, vlmax));
+        outer_sum += diff_i * inner_val;
+    }
+    *result = nk_f64_sqrt_rvv(outer_sum > 0 ? outer_sum : 0);
+}
+
+NK_PUBLIC void nk_mahalanobis_f64_rvv(nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t n,
+                                      nk_f64_t *result) {
+    nk_size_t vlmax = __riscv_vsetvlmax_e64m4();
+    vfloat64m1_t sum_f64m1 = __riscv_vfmv_v_f_f64m1(0.0, 1);
+    nk_f64_t outer_compensation = 0;
+    for (nk_size_t i = 0; i < n; ++i) {
+        nk_f64_t diff_i = a[i] - b[i];
+        vfloat64m4_t inner_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, vlmax);
+        vfloat64m4_t compensation_f64m4 = __riscv_vfmv_v_f_f64m4(0.0, vlmax);
+        nk_f64_t const *c_row = c + i * n;
+        nk_size_t remaining = n;
+        for (nk_size_t vector_length; remaining > 0; remaining -= vector_length, c_row += vector_length) {
+            vector_length = __riscv_vsetvl_e64m4(remaining);
+            nk_size_t j = n - remaining;
+            vfloat64m4_t vc_f64m4 = __riscv_vle64_v_f64m4(c_row, vector_length);
+            vfloat64m4_t va_f64m4 = __riscv_vle64_v_f64m4(a + j, vector_length);
+            vfloat64m4_t vb_f64m4 = __riscv_vle64_v_f64m4(b + j, vector_length);
+            vfloat64m4_t diff_j_f64m4 = __riscv_vfsub_vv_f64m4(va_f64m4, vb_f64m4, vector_length);
+            vfloat64m4_t product_f64m4 = __riscv_vfmul_vv_f64m4(vc_f64m4, diff_j_f64m4, vector_length);
+            vfloat64m4_t corrected_term_f64m4 = __riscv_vfsub_vv_f64m4(product_f64m4, compensation_f64m4,
+                                                                       vector_length);
+            vfloat64m4_t running_sum_f64m4 = __riscv_vfadd_vv_f64m4_tu(inner_f64m4, inner_f64m4, corrected_term_f64m4,
+                                                                       vector_length);
+            compensation_f64m4 = __riscv_vfsub_vv_f64m4_tu(
+                compensation_f64m4, __riscv_vfsub_vv_f64m4(running_sum_f64m4, inner_f64m4, vector_length),
+                corrected_term_f64m4, vector_length);
+            inner_f64m4 = running_sum_f64m4;
+        }
+        vfloat64m1_t zero_f64m1 = __riscv_vfmv_v_f_f64m1(0.0, 1);
+        nk_f64_t inner_val = __riscv_vfmv_f_s_f64m1_f64(
+            __riscv_vfredusum_vs_f64m4_f64m1(inner_f64m4, zero_f64m1, vlmax));
+        nk_f64_t product_outer = diff_i * inner_val;
+        nk_f64_t old_sum = __riscv_vfmv_f_s_f64m1_f64(sum_f64m1);
+        nk_f64_t new_sum = old_sum + product_outer;
+        if (nk_f64_abs_(old_sum) >= nk_f64_abs_(product_outer))
+            outer_compensation += (old_sum - new_sum) + product_outer;
+        else outer_compensation += (product_outer - new_sum) + old_sum;
+        sum_f64m1 = __riscv_vfmv_v_f_f64m1(new_sum, 1);
+    }
+    nk_f64_t quadratic = __riscv_vfmv_f_s_f64m1_f64(sum_f64m1) + outer_compensation;
+    *result = nk_f64_sqrt_rvv(quadratic > 0 ? quadratic : 0);
+}
+
+NK_PUBLIC void nk_mahalanobis_f16_rvv(nk_f16_t const *a, nk_f16_t const *b, nk_f16_t const *c, nk_size_t n,
+                                      nk_f32_t *result) {
+    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
+    vfloat32m1_t sum_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
+    for (nk_size_t i = 0; i < n; ++i) {
+        nk_f32_t a_i, b_i;
+        nk_f16_to_f32_serial(a + i, &a_i);
+        nk_f16_to_f32_serial(b + i, &b_i);
+        nk_f32_t diff_i = a_i - b_i;
+
+        vfloat32m2_t inner_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
+        nk_f16_t const *c_row = c + i * n;
+        nk_size_t remaining = n;
+        for (nk_size_t vector_length; remaining > 0; remaining -= vector_length, c_row += vector_length) {
+            vector_length = __riscv_vsetvl_e16m1(remaining);
+            nk_size_t j = n - remaining;
+            vuint16m1_t vc_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)c_row, vector_length);
+            vuint16m1_t va_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)(a + j), vector_length);
+            vuint16m1_t vb_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)(b + j), vector_length);
+            vfloat32m2_t vc_f32m2 = nk_f16m1_to_f32m2_rvv_(vc_u16m1, vector_length);
+            vfloat32m2_t va_f32m2 = nk_f16m1_to_f32m2_rvv_(va_u16m1, vector_length);
+            vfloat32m2_t vb_f32m2 = nk_f16m1_to_f32m2_rvv_(vb_u16m1, vector_length);
+            vfloat32m2_t diff_j_f32m2 = __riscv_vfsub_vv_f32m2(va_f32m2, vb_f32m2, vector_length);
+            inner_f32m2 = __riscv_vfmacc_vv_f32m2_tu(inner_f32m2, vc_f32m2, diff_j_f32m2, vector_length);
+        }
+        vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
+        nk_f32_t inner_val = __riscv_vfmv_f_s_f32m1_f32(
+            __riscv_vfredusum_vs_f32m2_f32m1(inner_f32m2, zero_f32m1, vlmax));
+        sum_f32m1 = __riscv_vfmv_v_f_f32m1(__riscv_vfmv_f_s_f32m1_f32(sum_f32m1) + diff_i * inner_val, 1);
+    }
+    nk_f32_t quadratic_f16 = __riscv_vfmv_f_s_f32m1_f32(sum_f32m1);
+    *result = nk_f32_sqrt_rvv(quadratic_f16 > 0 ? quadratic_f16 : 0);
+}
+
+NK_PUBLIC void nk_mahalanobis_bf16_rvv(nk_bf16_t const *a, nk_bf16_t const *b, nk_bf16_t const *c, nk_size_t n,
+                                       nk_f32_t *result) {
+    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
+    vfloat32m1_t sum_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
+    for (nk_size_t i = 0; i < n; ++i) {
+        nk_f32_t a_i, b_i;
+        nk_bf16_to_f32_serial(a + i, &a_i);
+        nk_bf16_to_f32_serial(b + i, &b_i);
+        nk_f32_t diff_i = a_i - b_i;
+
+        vfloat32m2_t inner_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
+        nk_bf16_t const *c_row = c + i * n;
+        nk_size_t remaining = n;
+        for (nk_size_t vector_length; remaining > 0; remaining -= vector_length, c_row += vector_length) {
+            vector_length = __riscv_vsetvl_e16m1(remaining);
+            nk_size_t j = n - remaining;
+            vuint16m1_t vc_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)c_row, vector_length);
+            vuint16m1_t va_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)(a + j), vector_length);
+            vuint16m1_t vb_u16m1 = __riscv_vle16_v_u16m1((nk_u16_t const *)(b + j), vector_length);
+            vfloat32m2_t vc_f32m2 = nk_bf16m1_to_f32m2_rvv_(vc_u16m1, vector_length);
+            vfloat32m2_t va_f32m2 = nk_bf16m1_to_f32m2_rvv_(va_u16m1, vector_length);
+            vfloat32m2_t vb_f32m2 = nk_bf16m1_to_f32m2_rvv_(vb_u16m1, vector_length);
+            vfloat32m2_t diff_j_f32m2 = __riscv_vfsub_vv_f32m2(va_f32m2, vb_f32m2, vector_length);
+            inner_f32m2 = __riscv_vfmacc_vv_f32m2_tu(inner_f32m2, vc_f32m2, diff_j_f32m2, vector_length);
+        }
+        vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
+        nk_f32_t inner_val = __riscv_vfmv_f_s_f32m1_f32(
+            __riscv_vfredusum_vs_f32m2_f32m1(inner_f32m2, zero_f32m1, vlmax));
+        sum_f32m1 = __riscv_vfmv_v_f_f32m1(__riscv_vfmv_f_s_f32m1_f32(sum_f32m1) + diff_i * inner_val, 1);
+    }
+    nk_f32_t quadratic_bf16 = __riscv_vfmv_f_s_f32m1_f32(sum_f32m1);
+    *result = nk_f32_sqrt_rvv(quadratic_bf16 > 0 ? quadratic_bf16 : 0);
+}
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#endif // NK_TARGET_RVV
+#endif // NK_TARGET_RISCV_
+#endif // NK_CURVED_RVV_H

package/include/numkong/curved/serial.h

@@ -0,0 +1,207 @@
+/**
+ *  @brief SWAR-accelerated Curved Space Similarity for SIMD-free CPUs.
+ *  @file include/numkong/curved/serial.h
+ *  @author Ash Vardanian
+ *  @date January 14, 2026
+ *
+ *  @sa include/numkong/curved.h
+ *
+ *  Implements bilinear forms and Mahalanobis distance with precision-appropriate strategies:
+ *  - f64 inputs use Dot2 (Ogita-Rump-Oishi 2005) for error-free transformations
+ *  - f32/f16/bf16 inputs upcast to wider accumulators (f64/f32), providing sufficient
+ *    precision headroom without compensation overhead
+ *
+ *  Bilinear form: aᵀ × C × b = Σᵢ aᵢ × (Σⱼ cᵢⱼ × bⱼ)
+ *
+ *  The nested loop structure has two accumulation levels:
+ *  - Inner: Σⱼ cᵢⱼ × bⱼ (O(n) terms per row)
+ *  - Outer: Σᵢ aᵢ × inner_result (O(n) terms total)
+ *
+ *  For f64→f64 (no upcast headroom): Dot2 uses TwoProd and TwoSum error-free
+ *  transformations at both levels, capturing rounding errors in compensation terms.
+ *
+ *  For upcasted types (f32→f64, f16→f32, bf16→f32): the wider accumulator provides
+ *  enough extra mantissa bits that simple accumulation suffices.
+ *
+ *  @see Ogita, T., Rump, S.M., Oishi, S. (2005). "Accurate Sum and Dot Product"
+ */
+#ifndef NK_CURVED_SERIAL_H
+#define NK_CURVED_SERIAL_H
+
+#include "numkong/types.h"
+#include "numkong/spatial/serial.h" // `nk_f64_sqrt_serial`
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/**
+ *  @brief Macro for bilinear form aᵀ × C × b with simple accumulation.
+ *
+ *  Suitable for upcasted types where the wider accumulator provides sufficient
+ *  precision headroom (f32→f64, f16→f32, bf16→f32).
+ */
+#define nk_define_bilinear_(input_type, accumulator_type, output_type, load_and_convert)                         \
+    NK_PUBLIC void nk_bilinear_##input_type##_serial(nk_##input_type##_t const *a, nk_##input_type##_t const *b, \
+                                                     nk_##input_type##_t const *c, nk_size_t n,                  \
+                                                     nk_##output_type##_t *result) {                             \
+        nk_##accumulator_type##_t outer_sum = 0;                                                                 \
+        nk_##accumulator_type##_t vector_a_value, vector_b_value, tensor_value;                                  \
+        for (nk_size_t row = 0; row != n; ++row) {                                                               \
+            nk_##accumulator_type##_t inner_sum = 0;                                                             \
+            load_and_convert(a + row, &vector_a_value);                                                          \
+            for (nk_size_t column = 0; column != n; ++column) {                                                  \
+                load_and_convert(b + column, &vector_b_value);                                                   \
+                load_and_convert(c + row * n + column, &tensor_value);                                           \
+                inner_sum += tensor_value * vector_b_value;                                                      \
+            }                                                                                                    \
+            outer_sum += vector_a_value * inner_sum;                                                             \
+        }                                                                                                        \
+        *result = (nk_##output_type##_t)(outer_sum);                                                             \
+    }
+
+/**
+ *  @brief Macro for complex bilinear form aᵀ × C × b with simple accumulation.
+ *
+ *  Suitable for upcasted complex types where the wider accumulator provides
+ *  sufficient precision headroom.
+ */
+#define nk_define_bilinear_complex_(input_type, accumulator_type, output_type, load_and_convert)                    \
+    NK_PUBLIC void nk_bilinear_##input_type##_serial(                                                               \
+        nk_##input_type##_t const *a_pairs, nk_##input_type##_t const *b_pairs, nk_##input_type##_t const *c_pairs, \
+        nk_size_t n, nk_##output_type##c_t *results) {                                                              \
+        nk_##accumulator_type##_t outer_sum_real = 0, outer_sum_imag = 0;                                           \
+        nk_##accumulator_type##_t a_real, a_imag, b_real, b_imag, c_real, c_imag;                                   \
+        for (nk_size_t row = 0; row != n; ++row) {                                                                  \
+            nk_##accumulator_type##_t inner_sum_real = 0, inner_sum_imag = 0;                                       \
+            load_and_convert(&(a_pairs + row)->real, &a_real);                                                      \
+            load_and_convert(&(a_pairs + row)->imag, &a_imag);                                                      \
+            for (nk_size_t column = 0; column != n; ++column) {                                                     \
+                load_and_convert(&(b_pairs + column)->real, &b_real);                                               \
+                load_and_convert(&(b_pairs + column)->imag, &b_imag);                                               \
+                load_and_convert(&(c_pairs + row * n + column)->real, &c_real);                                     \
+                load_and_convert(&(c_pairs + row * n + column)->imag, &c_imag);                                     \
+                inner_sum_real += c_real * b_real - c_imag * b_imag;                                                \
+                inner_sum_imag += c_real * b_imag + c_imag * b_real;                                                \
+            }                                                                                                       \
+            /* Complex multiply: a_i * inner_result */                                                              \
+            outer_sum_real += a_real * inner_sum_real - a_imag * inner_sum_imag;                                    \
+            outer_sum_imag += a_real * inner_sum_imag + a_imag * inner_sum_real;                                    \
+        }                                                                                                           \
+        results->real = outer_sum_real;                                                                             \
+        results->imag = outer_sum_imag;                                                                             \
+    }
+
+/**
+ *  @brief Macro for Mahalanobis distance √((a−b)ᵀ × C × (a−b)) with simple accumulation.
+ *
+ *  Suitable for upcasted types where the wider accumulator provides sufficient
+ *  precision headroom. Differences are computed in the accumulator precision.
+ */
+#define nk_define_mahalanobis_(input_type, accumulator_type, output_type, load_and_convert)                         \
+    NK_PUBLIC void nk_mahalanobis_##input_type##_serial(nk_##input_type##_t const *a, nk_##input_type##_t const *b, \
+                                                        nk_##input_type##_t const *c, nk_size_t n,                  \
+                                                        nk_##output_type##_t *result) {                             \
+        nk_##accumulator_type##_t outer_sum = 0;                                                                    \
+        nk_##accumulator_type##_t a_row_value, b_row_value, a_column_value, b_column_value, tensor_value;           \
+        for (nk_size_t row = 0; row != n; ++row) {                                                                  \
+            nk_##accumulator_type##_t inner_sum = 0;                                                                \
+            load_and_convert(a + row, &a_row_value);                                                                \
+            load_and_convert(b + row, &b_row_value);                                                                \
+            nk_##accumulator_type##_t difference_row = a_row_value - b_row_value;                                   \
+            for (nk_size_t column = 0; column != n; ++column) {                                                     \
+                load_and_convert(a + column, &a_column_value);                                                      \
+                load_and_convert(b + column, &b_column_value);                                                      \
+                load_and_convert(c + row * n + column, &tensor_value);                                              \
+                nk_##accumulator_type##_t difference_column = a_column_value - b_column_value;                      \
+                inner_sum += tensor_value * difference_column;                                                      \
+            }                                                                                                       \
+            outer_sum += difference_row * inner_sum;                                                                \
+        }                                                                                                           \
+        nk_##accumulator_type##_t quadratic = outer_sum;                                                            \
+        *result = nk_##accumulator_type##_sqrt_serial(quadratic > 0 ? quadratic : 0);                               \
+    }
+
+// f32 → f64 accumulator → f64 output
+nk_define_bilinear_(f32, f64, f64, nk_assign_from_to_)          // nk_bilinear_f32_serial
+nk_define_bilinear_complex_(f32c, f64, f64, nk_assign_from_to_) // nk_bilinear_f32c_serial
+nk_define_mahalanobis_(f32, f64, f64, nk_assign_from_to_)       // nk_mahalanobis_f32_serial
+
+// f16 → f32 accumulator → f32 output: f32 provides ample headroom for f16 (~3 vs ~7 decimal digits)
+nk_define_bilinear_(f16, f32, f32, nk_f16_to_f32_serial)          // nk_bilinear_f16_serial
+nk_define_bilinear_complex_(f16c, f32, f32, nk_f16_to_f32_serial) // nk_bilinear_f16c_serial
+nk_define_mahalanobis_(f16, f32, f32, nk_f16_to_f32_serial)       // nk_mahalanobis_f16_serial
+
+// bf16 → f32 accumulator → f32 output: f32 provides ample headroom for bf16 (~2 vs ~7 decimal digits)
+nk_define_bilinear_(bf16, f32, f32, nk_bf16_to_f32_serial)          // nk_bilinear_bf16_serial
+nk_define_bilinear_complex_(bf16c, f32, f32, nk_bf16_to_f32_serial) // nk_bilinear_bf16c_serial
+nk_define_mahalanobis_(bf16, f32, f32, nk_bf16_to_f32_serial)       // nk_mahalanobis_bf16_serial
+
+#undef nk_define_bilinear_
+#undef nk_define_bilinear_complex_
+#undef nk_define_mahalanobis_
+
+NK_PUBLIC void nk_bilinear_f64_serial(nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t n,
+                                      nk_f64_t *result) {
+    nk_f64_t outer_sum = 0, outer_comp = 0;
+    for (nk_size_t row = 0; row != n; ++row) {
+        nk_f64_t inner_sum = 0, inner_comp = 0;
+        for (nk_size_t col = 0; col != n; ++col) nk_f64_dot2_(&inner_sum, &inner_comp, c[row * n + col], b[col]);
+        nk_f64_t cb_j = inner_sum + inner_comp;
+        nk_f64_dot2_(&outer_sum, &outer_comp, a[row], cb_j);
+    }
+    *result = outer_sum + outer_comp;
+}
+
+NK_PUBLIC void nk_bilinear_f64c_serial(nk_f64c_t const *a_pairs, nk_f64c_t const *b_pairs, nk_f64c_t const *c_pairs,
+                                       nk_size_t n, nk_f64c_t *results) {
+    nk_f64_t outer_sum_real = 0, outer_comp_real = 0;
+    nk_f64_t outer_sum_imag = 0, outer_comp_imag = 0;
+    for (nk_size_t row = 0; row != n; ++row) {
+        nk_f64_t a_real = a_pairs[row].real;
+        nk_f64_t a_imag = a_pairs[row].imag;
+        // 4 Dot2 accumulators for inner cross-terms
+        nk_f64_t sum_rr = 0, comp_rr = 0;
+        nk_f64_t sum_ii = 0, comp_ii = 0;
+        nk_f64_t sum_ri = 0, comp_ri = 0;
+        nk_f64_t sum_ir = 0, comp_ir = 0;
+        for (nk_size_t col = 0; col != n; ++col) {
+            nk_f64_t b_real = b_pairs[col].real, b_imag = b_pairs[col].imag;
+            nk_f64_t c_real = c_pairs[row * n + col].real, c_imag = c_pairs[row * n + col].imag;
+            nk_f64_dot2_(&sum_rr, &comp_rr, c_real, b_real);
+            nk_f64_dot2_(&sum_ii, &comp_ii, c_imag, b_imag);
+            nk_f64_dot2_(&sum_ri, &comp_ri, c_real, b_imag);
+            nk_f64_dot2_(&sum_ir, &comp_ir, c_imag, b_real);
+        }
+        nk_f64_t inner_real = (sum_rr + comp_rr) - (sum_ii + comp_ii);
+        nk_f64_t inner_imag = (sum_ri + comp_ri) + (sum_ir + comp_ir);
+        // Outer Dot2 complex multiply: a × inner
+        nk_f64_dot2_(&outer_sum_real, &outer_comp_real, a_real, inner_real);
+        nk_f64_dot2_(&outer_sum_real, &outer_comp_real, -a_imag, inner_imag);
+        nk_f64_dot2_(&outer_sum_imag, &outer_comp_imag, a_real, inner_imag);
+        nk_f64_dot2_(&outer_sum_imag, &outer_comp_imag, a_imag, inner_real);
+    }
+    results->real = outer_sum_real + outer_comp_real;
+    results->imag = outer_sum_imag + outer_comp_imag;
+}
+
+NK_PUBLIC void nk_mahalanobis_f64_serial(nk_f64_t const *a, nk_f64_t const *b, nk_f64_t const *c, nk_size_t n,
+                                         nk_f64_t *result) {
+    nk_f64_t outer_sum = 0, outer_comp = 0;
+    for (nk_size_t row = 0; row != n; ++row) {
+        nk_f64_t diff_row = a[row] - b[row];
+        nk_f64_t inner_sum = 0, inner_comp = 0;
+        for (nk_size_t col = 0; col != n; ++col)
+            nk_f64_dot2_(&inner_sum, &inner_comp, c[row * n + col], a[col] - b[col]);
+        nk_f64_t cb_j = inner_sum + inner_comp;
+        nk_f64_dot2_(&outer_sum, &outer_comp, diff_row, cb_j);
+    }
+    nk_f64_t quadratic = outer_sum + outer_comp;
+    *result = nk_f64_sqrt_serial(quadratic > 0 ? quadratic : 0);
+}
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_CURVED_SERIAL_H