numkong 7.0.0

package/include/numkong/cast/v128relaxed.h

@@ -0,0 +1,180 @@
+/**
+ *  @brief SIMD-accelerated Type Conversions for WASM.
+ *  @file include/numkong/cast/v128relaxed.h
+ */
+
+#ifndef NK_CAST_V128RELAXED_H
+#define NK_CAST_V128RELAXED_H
+
+#if NK_TARGET_V128RELAXED
+
+#include "numkong/types.h"
+#include "numkong/cast/serial.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("relaxed-simd"))), apply_to = function)
+#endif
+
+/** @brief Native WASM SIMD 128-bit load. */
+NK_INTERNAL void nk_load_b128_v128relaxed_(void const *src, nk_b128_vec_t *dst) { dst->v128 = wasm_v128_load(src); }
+/** @brief Native WASM SIMD 256-bit load using two v128 loads. */
+NK_INTERNAL void nk_load_b256_v128relaxed_(void const *src, nk_b256_vec_t *dst) {
+    dst->v128s[0] = wasm_v128_load(src);
+    dst->v128s[1] = wasm_v128_load((char const *)src + 16);
+}
+/** @brief Native WASM SIMD 128-bit store. */
+NK_INTERNAL void nk_store_b128_v128relaxed_(nk_b128_vec_t const *src, void *dst) { wasm_v128_store(dst, src->v128); }
+/** @brief Native WASM SIMD 256-bit store using two v128 stores. */
+NK_INTERNAL void nk_store_b256_v128relaxed_(nk_b256_vec_t const *src, void *dst) {
+    wasm_v128_store(dst, src->v128s[0]);
+    wasm_v128_store((char *)dst + 16, src->v128s[1]);
+}
+
+/** @brief BF16 is the upper 16 bits of F32, so zero-extend to u32 and shift left by 16. */
+NK_INTERNAL nk_b128_vec_t nk_bf16x4_to_f32x4_v128relaxed_(nk_b64_vec_t bf16_vec) {
+    v128_t bf16_u16x4_in_u64 = wasm_v128_load64_zero(&bf16_vec.u64);
+    v128_t bf16_u32x4_low = wasm_u32x4_extend_low_u16x8(bf16_u16x4_in_u64);
+    nk_b128_vec_t result;
+    result.v128 = wasm_i32x4_shl(bf16_u32x4_low, 16);
+    return result;
+}
+
+/**
+ *  @brief F16→F32: extract sign/exp/mantissa, rebias exponent (F16 bias=15, F32 bias=127, delta=112),
+ *  widen mantissa from 10 to 23 bits.  Early-exit when all lanes are normal (exp in [1,30]),
+ *  skipping the expensive f32x4.convert_u32x4 needed for denormal FPU-based normalization.
+ */
+NK_INTERNAL nk_b128_vec_t nk_f16x4_to_f32x4_v128relaxed_(nk_b64_vec_t f16_vec) {
+    v128_t f16_u16x4_in_u64 = wasm_v128_load64_zero(&f16_vec.u64);
+    v128_t f16_u32x4 = wasm_u32x4_extend_low_u16x8(f16_u16x4_in_u64);
+
+    v128_t sign_u32x4 = wasm_v128_and(f16_u32x4, wasm_i32x4_splat(0x8000));                  // bit 15
+    v128_t exp_u32x4 = wasm_v128_and(wasm_u32x4_shr(f16_u32x4, 10), wasm_i32x4_splat(0x1F)); // bits 14-10
+    v128_t mant_u32x4 = wasm_v128_and(f16_u32x4, wasm_i32x4_splat(0x03FF));                  // bits 9-0
+
+    v128_t sign_f32_u32x4 = wasm_i32x4_shl(sign_u32x4, 16); // shift sign to F32 bit 31
+
+    // Normal path: rebias exponent, widen mantissa
+    v128_t exp_rebiased_u32x4 = wasm_i32x4_add(exp_u32x4, wasm_i32x4_splat(112));
+    v128_t normal_exp_u32x4 = wasm_i32x4_shl(exp_rebiased_u32x4, 23);
+    v128_t normal_mant_u32x4 = wasm_i32x4_shl(mant_u32x4, 13);
+    v128_t normal_bits_u32x4 = wasm_v128_or(sign_f32_u32x4, wasm_v128_or(normal_exp_u32x4, normal_mant_u32x4));
+
+    // Early exit: skip zero/denormal/inf/NaN handling when all lanes are normal
+    v128_t exp_zero_mask = wasm_i32x4_eq(exp_u32x4, wasm_i32x4_splat(0));
+    v128_t exp_max_mask = wasm_i32x4_eq(exp_u32x4, wasm_i32x4_splat(31));
+    v128_t exceptional_mask = wasm_v128_or(exp_zero_mask, exp_max_mask);
+    if (!wasm_v128_any_true(exceptional_mask)) {
+        nk_b128_vec_t result;
+        result.v128 = normal_bits_u32x4;
+        return result;
+    }
+
+    // Slow path: handle zero (exp=0, mant=0), denormal (exp=0, mant!=0), inf/NaN (exp=31)
+    v128_t zero_bits_u32x4 = sign_f32_u32x4;
+    v128_t inf_nan_bits_u32x4 = wasm_v128_or(
+        sign_f32_u32x4, wasm_v128_or(wasm_i32x4_splat(0x7F800000), wasm_i32x4_shl(mant_u32x4, 13)));
+
+    // Denormals: convert mantissa to f32 and multiply by 2^-24, letting the FPU normalize.
+    // This avoids a manual CLZ+shift loop.  The f32x4.convert_u32x4 legalizes to a
+    // multi-instruction sequence on x86 (no native u32→f32 until AVX-512), which is why
+    // the early exit above is so valuable.
+    v128_t mant_f32x4 = wasm_f32x4_convert_u32x4(mant_u32x4);
+    v128_t denorm_normalized_f32x4 = wasm_f32x4_mul(mant_f32x4, wasm_f32x4_splat(0x1p-24f));
+    v128_t denorm_bits_u32x4 = wasm_v128_or(denorm_normalized_f32x4, sign_f32_u32x4);
+
+    v128_t mant_zero_mask = wasm_i32x4_eq(mant_u32x4, wasm_i32x4_splat(0));
+    v128_t is_zero_mask = wasm_v128_and(exp_zero_mask, mant_zero_mask);
+    v128_t is_denormal_mask = wasm_v128_andnot(exp_zero_mask, mant_zero_mask);
+
+    // Blend via relaxed_laneselect (1 instruction: vblendvps on x86, vs 3 for and/andn/or)
+    v128_t result_u32x4 = normal_bits_u32x4;
+    result_u32x4 = wasm_i32x4_relaxed_laneselect(zero_bits_u32x4, result_u32x4, is_zero_mask);
+    result_u32x4 = wasm_i32x4_relaxed_laneselect(denorm_bits_u32x4, result_u32x4, is_denormal_mask);
+    result_u32x4 = wasm_i32x4_relaxed_laneselect(inf_nan_bits_u32x4, result_u32x4, exp_max_mask);
+
+    nk_b128_vec_t result;
+    result.v128 = result_u32x4;
+    return result;
+}
+
+/**
+ *  @brief E4M3→F32: 4-bit exponent (bias=7→127, delta=120), 3-bit mantissa (shift by 20).
+ *  Subnormal via FPU: mant * (1/512) = mant * 2^-9.  NaN only at exp=15,mant=7.
+ */
+NK_INTERNAL nk_b128_vec_t nk_e4m3x4_to_f32x4_v128relaxed_(nk_b32_vec_t e4m3_vec) {
+    v128_t e4m3_u32x4 = wasm_u32x4_extend_low_u16x8(wasm_u16x8_extend_low_u8x16(wasm_v128_load32_zero(&e4m3_vec.u32)));
+    v128_t exp_u32x4 = wasm_v128_and(wasm_u32x4_shr(e4m3_u32x4, 3), wasm_i32x4_splat(0x0F));
+    v128_t mant_u32x4 = wasm_v128_and(e4m3_u32x4, wasm_i32x4_splat(0x07));
+    v128_t sign_u32x4 = wasm_i32x4_shl(wasm_u32x4_shr(e4m3_u32x4, 7), 31);
+    v128_t f32_exp_u32x4 = wasm_i32x4_shl(wasm_i32x4_add(exp_u32x4, wasm_i32x4_splat(120)), 23);
+    v128_t f32_mant_u32x4 = wasm_i32x4_shl(mant_u32x4, 20);
+    v128_t normal_bits_u32x4 = wasm_v128_or(sign_u32x4, wasm_v128_or(f32_exp_u32x4, f32_mant_u32x4));
+    v128_t subnorm_abs_f32x4 = wasm_f32x4_mul(wasm_f32x4_convert_u32x4(mant_u32x4), wasm_f32x4_splat(1.0f / 512.0f));
+    v128_t subnorm_f32x4 = wasm_v128_or(subnorm_abs_f32x4, sign_u32x4);
+    v128_t exp_zero_mask = wasm_i32x4_eq(exp_u32x4, wasm_i32x4_splat(0));
+    v128_t is_nan_mask = wasm_v128_and(wasm_i32x4_eq(exp_u32x4, wasm_i32x4_splat(15)),
+                                       wasm_i32x4_eq(mant_u32x4, wasm_i32x4_splat(7)));
+    v128_t exceptional_mask = wasm_v128_or(exp_zero_mask, is_nan_mask);
+    if (!wasm_v128_any_true(exceptional_mask)) {
+        nk_b128_vec_t result;
+        result.v128 = normal_bits_u32x4;
+        return result;
+    }
+    v128_t result_u32x4 = wasm_i32x4_relaxed_laneselect(subnorm_f32x4, normal_bits_u32x4, exp_zero_mask);
+    if (wasm_v128_any_true(is_nan_mask)) {
+        v128_t nan_bits = wasm_v128_or(sign_u32x4, wasm_i32x4_splat(0x7FC00000));
+        result_u32x4 = wasm_i32x4_relaxed_laneselect(nan_bits, result_u32x4, is_nan_mask);
+    }
+    nk_b128_vec_t result;
+    result.v128 = result_u32x4;
+    return result;
+}
+
+/**
+ *  @brief E5M2→F32: same exponent encoding as F16 (5-bit, bias=15, delta=112), 2-bit mantissa (shift by 21).
+ *  Subnormal via FPU: mant * (1/65536) = mant * 2^-16.  Inf at exp=31,mant=0; NaN otherwise.
+ */
+NK_INTERNAL nk_b128_vec_t nk_e5m2x4_to_f32x4_v128relaxed_(nk_b32_vec_t e5m2_vec) {
+    v128_t e5m2_u32x4 = wasm_u32x4_extend_low_u16x8(wasm_u16x8_extend_low_u8x16(wasm_v128_load32_zero(&e5m2_vec.u32)));
+    v128_t exp_u32x4 = wasm_v128_and(wasm_u32x4_shr(e5m2_u32x4, 2), wasm_i32x4_splat(0x1F));
+    v128_t mant_u32x4 = wasm_v128_and(e5m2_u32x4, wasm_i32x4_splat(0x03));
+    v128_t sign_u32x4 = wasm_i32x4_shl(wasm_u32x4_shr(e5m2_u32x4, 7), 31);
+    v128_t f32_exp_u32x4 = wasm_i32x4_shl(wasm_i32x4_add(exp_u32x4, wasm_i32x4_splat(112)), 23);
+    v128_t f32_mant_u32x4 = wasm_i32x4_shl(mant_u32x4, 21);
+    v128_t normal_bits_u32x4 = wasm_v128_or(sign_u32x4, wasm_v128_or(f32_exp_u32x4, f32_mant_u32x4));
+    v128_t subnorm_abs_f32x4 = wasm_f32x4_mul(wasm_f32x4_convert_u32x4(mant_u32x4), wasm_f32x4_splat(1.0f / 65536.0f));
+    v128_t subnorm_f32x4 = wasm_v128_or(subnorm_abs_f32x4, sign_u32x4);
+    v128_t exp_zero_mask = wasm_i32x4_eq(exp_u32x4, wasm_i32x4_splat(0));
+    v128_t exp_max_mask = wasm_i32x4_eq(exp_u32x4, wasm_i32x4_splat(31));
+    v128_t exceptional_mask = wasm_v128_or(exp_zero_mask, exp_max_mask);
+    if (!wasm_v128_any_true(exceptional_mask)) {
+        nk_b128_vec_t result;
+        result.v128 = normal_bits_u32x4;
+        return result;
+    }
+    v128_t result_u32x4 = wasm_i32x4_relaxed_laneselect(subnorm_f32x4, normal_bits_u32x4, exp_zero_mask);
+    v128_t mant_zero_mask = wasm_i32x4_eq(mant_u32x4, wasm_i32x4_splat(0));
+    v128_t inf_bits_u32x4 = wasm_v128_or(sign_u32x4, wasm_i32x4_splat(0x7F800000));
+    v128_t nan_bits_u32x4 = wasm_v128_or(sign_u32x4, wasm_i32x4_splat(0x7FC00000));
+    v128_t special_bits_u32x4 = wasm_i32x4_relaxed_laneselect(inf_bits_u32x4, nan_bits_u32x4, mant_zero_mask);
+    result_u32x4 = wasm_i32x4_relaxed_laneselect(special_bits_u32x4, result_u32x4, exp_max_mask);
+    nk_b128_vec_t result;
+    result.v128 = result_u32x4;
+    return result;
+}
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_V128RELAXED
+#endif // NK_CAST_V128RELAXED_H

package/include/numkong/cast.h

@@ -0,0 +1,230 @@
+/**
+ *  @brief SIMD-accelerated Type Conversions.
+ *  @file include/numkong/cast.h
+ *  @author Ash Vardanian
+ *  @date January 2, 2026
+ *
+ *  This file focuses on numeric types not uniformly supported across platforms, prioritizing:
+ *
+ *  - `e5m2` & `e4m3` ↔ `f16` & `bf16` - used for low-precision dot-products on modern CPUs,
+ *  - `e5m2` & `e4m3` ↔ `f32` - used for low-precision dot-products on older CPUs,
+ *  - `f16` & `bf16` ↔ `f32` - often used for half-precision dot-products on older CPUs,
+ *
+ *  Unlike most operation classes in NumKong, these are dependent on two input types: "from" & "to".
+ *  It contains scalar helpers named like `nk_f16_to_f32_serial_` as well as buffer-to-buffer
+ *  `memcpy`-like vectorized operations, such as `nk_cast_f16_to_f32` with `nk_cast_f16_to_f32_serial`,
+ *  `nk_cast_f16_to_f32_neon`, `nk_cast_f16_to_f32_skylake`, and other platform-specific variants.
+ *
+ *  It also includes "partial load" and "partial store" type-punned helper functions for handling
+ *  IO between memory and registers, that are extensively reused in reductions, elementwise ops, and
+ *  dot-products.
+ *
+ *  Float-format narrowing uses round-to-nearest, ties-to-even. Float-to-integer narrowing follows
+ *  the same tie rule, saturates infinities, and maps NaNs to zero.
+ *
+ *  Assuming the overall breadth and sparsity of our type system, its clear, that not all type conversions
+ *  have equivalent relevance. With ~16 numeric types we'd be looking at 21x21=441 conversions for:
+ *
+ *              e4m3    e5m2    bf16    f16     f32     f64
+ *                              bf16c   f16c    f32c    f64c
+ *              i4      i8              i16     i32     i64
+ *      u1      u4      u8              u16     u32     u64
+ *
+ *  To simplify the design and make it more broadly applicable in AI workloads, we implement a slower
+ *  @b "hub-and-spoke" design to guiding most conversions through an intermediate type, like `f64` or `i64`.
+ *
+ */
+#ifndef NK_CAST_H
+#define NK_CAST_H
+
+#include "numkong/types.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/**
+ *  @brief Elementwise type-casting for arrays of entries.
+ *
+ *  @param[in] from The immutable input source array containing `n` elements of `from_type` type.
+ *  @param[in] from_type The type of elements in the immutable source array.
+ *  @param[in] n The number of elements in both input and output arrays.
+ *  @param[in] to The mutable output array containing `n` elements of `to_type` type.
+ *  @param[in] to_type The type of elements in the mutable target array.
+ */
+NK_DYNAMIC void nk_cast(void const *from, nk_dtype_t from_type, nk_size_t n, void *to, nk_dtype_t to_type);
+
+/** @copydoc nk_cast */
+NK_PUBLIC void nk_cast_serial(void const *from, nk_dtype_t from_type, nk_size_t n, void *to, nk_dtype_t to_type);
+
+/** @brief Scalar conversion from f16 to f32. */
+NK_DYNAMIC void nk_f16_to_f32(nk_f16_t const *src, nk_f32_t *dest);
+/** @brief Scalar conversion from bf16 to f32. */
+NK_DYNAMIC void nk_bf16_to_f32(nk_bf16_t const *src, nk_f32_t *dest);
+/** @brief Scalar conversion from e4m3 to f32. */
+NK_DYNAMIC void nk_e4m3_to_f32(nk_e4m3_t const *src, nk_f32_t *dest);
+/** @brief Scalar conversion from e5m2 to f32. */
+NK_DYNAMIC void nk_e5m2_to_f32(nk_e5m2_t const *src, nk_f32_t *dest);
+/** @brief Scalar conversion from e2m3 to f32. */
+NK_DYNAMIC void nk_e2m3_to_f32(nk_e2m3_t const *src, nk_f32_t *dest);
+/** @brief Scalar conversion from e3m2 to f32. */
+NK_DYNAMIC void nk_e3m2_to_f32(nk_e3m2_t const *src, nk_f32_t *dest);
+
+/** @brief Scalar conversion from f32 to f16. */
+NK_DYNAMIC void nk_f32_to_f16(nk_f32_t const *src, nk_f16_t *dest);
+/** @brief Scalar conversion from f32 to bf16. */
+NK_DYNAMIC void nk_f32_to_bf16(nk_f32_t const *src, nk_bf16_t *dest);
+/** @brief Scalar conversion from f32 to e4m3. */
+NK_DYNAMIC void nk_f32_to_e4m3(nk_f32_t const *src, nk_e4m3_t *dest);
+/** @brief Scalar conversion from f32 to e5m2. */
+NK_DYNAMIC void nk_f32_to_e5m2(nk_f32_t const *src, nk_e5m2_t *dest);
+/** @brief Scalar conversion from f32 to e2m3. */
+NK_DYNAMIC void nk_f32_to_e2m3(nk_f32_t const *src, nk_e2m3_t *dest);
+/** @brief Scalar conversion from f32 to e3m2. */
+NK_DYNAMIC void nk_f32_to_e3m2(nk_f32_t const *src, nk_e3m2_t *dest);
+
+/** @copydoc nk_f16_to_f32 */
+NK_PUBLIC void nk_f16_to_f32_serial(nk_f16_t const *src, nk_f32_t *dest);
+/** @copydoc nk_f32_to_f16 */
+NK_PUBLIC void nk_f32_to_f16_serial(nk_f32_t const *src, nk_f16_t *dest);
+/** @copydoc nk_bf16_to_f32 */
+NK_PUBLIC void nk_bf16_to_f32_serial(nk_bf16_t const *src, nk_f32_t *dest);
+/** @copydoc nk_f32_to_bf16 */
+NK_PUBLIC void nk_f32_to_bf16_serial(nk_f32_t const *src, nk_bf16_t *dest);
+/** @copydoc nk_e4m3_to_f32 */
+NK_PUBLIC void nk_e4m3_to_f32_serial(nk_e4m3_t const *src, nk_f32_t *dest);
+/** @copydoc nk_f32_to_e4m3 */
+NK_PUBLIC void nk_f32_to_e4m3_serial(nk_f32_t const *src, nk_e4m3_t *dest);
+/** @copydoc nk_e5m2_to_f32 */
+NK_PUBLIC void nk_e5m2_to_f32_serial(nk_e5m2_t const *src, nk_f32_t *dest);
+/** @copydoc nk_f32_to_e5m2 */
+NK_PUBLIC void nk_f32_to_e5m2_serial(nk_f32_t const *src, nk_e5m2_t *dest);
+/** @copydoc nk_e2m3_to_f32 */
+NK_PUBLIC void nk_e2m3_to_f32_serial(nk_e2m3_t const *src, nk_f32_t *dest);
+/** @copydoc nk_f32_to_e2m3 */
+NK_PUBLIC void nk_f32_to_e2m3_serial(nk_f32_t const *src, nk_e2m3_t *dest);
+/** @copydoc nk_e3m2_to_f32 */
+NK_PUBLIC void nk_e3m2_to_f32_serial(nk_e3m2_t const *src, nk_f32_t *dest);
+/** @copydoc nk_f32_to_e3m2 */
+NK_PUBLIC void nk_f32_to_e3m2_serial(nk_f32_t const *src, nk_e3m2_t *dest);
+
+#if NK_TARGET_NEON
+/** @copydoc nk_cast */
+NK_PUBLIC void nk_cast_neon(void const *from, nk_dtype_t from_type, nk_size_t n, void *to, nk_dtype_t to_type);
+/** @copydoc nk_f16_to_f32 */
+NK_PUBLIC void nk_f16_to_f32_neon(nk_f16_t const *src, nk_f32_t *dest);
+/** @copydoc nk_f32_to_f16 */
+NK_PUBLIC void nk_f32_to_f16_neon(nk_f32_t const *src, nk_f16_t *dest);
+#endif // NK_TARGET_NEON
+
+#if NK_TARGET_HASWELL
+/** @copydoc nk_cast */
+NK_PUBLIC void nk_cast_haswell(void const *from, nk_dtype_t from_type, nk_size_t n, void *to, nk_dtype_t to_type);
+/** @copydoc nk_f16_to_f32 */
+NK_PUBLIC void nk_f16_to_f32_haswell(nk_f16_t const *src, nk_f32_t *dest);
+/** @copydoc nk_f32_to_f16 */
+NK_PUBLIC void nk_f32_to_f16_haswell(nk_f32_t const *src, nk_f16_t *dest);
+#endif // NK_TARGET_HASWELL
+
+#if NK_TARGET_SKYLAKE
+/** @copydoc nk_cast */
+NK_PUBLIC void nk_cast_skylake(void const *from, nk_dtype_t from_type, nk_size_t n, void *to, nk_dtype_t to_type);
+#endif // NK_TARGET_SKYLAKE
+
+#if NK_TARGET_ICELAKE
+/** @copydoc nk_cast */
+NK_PUBLIC void nk_cast_icelake(void const *from, nk_dtype_t from_type, nk_size_t n, void *to, nk_dtype_t to_type);
+#endif // NK_TARGET_ICELAKE
+
+#if NK_TARGET_SAPPHIRE
+/** @copydoc nk_cast */
+NK_PUBLIC void nk_cast_sapphire(void const *from, nk_dtype_t from_type, nk_size_t n, void *to, nk_dtype_t to_type);
+/** @copydoc nk_f16_to_f32 */
+NK_PUBLIC void nk_f16_to_f32_sapphire(nk_f16_t const *src, nk_f32_t *dest);
+/** @copydoc nk_f32_to_f16 */
+NK_PUBLIC void nk_f32_to_f16_sapphire(nk_f32_t const *src, nk_f16_t *dest);
+#endif // NK_TARGET_SAPPHIRE
+
+#if NK_TARGET_RVV
+/** @copydoc nk_cast */
+NK_PUBLIC void nk_cast_rvv(void const *from, nk_dtype_t from_type, nk_size_t n, void *to, nk_dtype_t to_type);
+#endif // NK_TARGET_RVV
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#include "numkong/cast/serial.h"
+#include "numkong/cast/neon.h"
+#include "numkong/cast/haswell.h"
+#include "numkong/cast/skylake.h"
+#include "numkong/cast/icelake.h"
+#include "numkong/cast/sapphire.h"
+#include "numkong/cast/rvv.h"
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if !NK_DYNAMIC_DISPATCH
+
+NK_PUBLIC void nk_cast(void const *from, nk_dtype_t from_type, nk_size_t n, void *to, nk_dtype_t to_type) {
+#if NK_TARGET_SAPPHIRE
+    nk_cast_sapphire(from, from_type, n, to, to_type);
+#elif NK_TARGET_ICELAKE
+    nk_cast_icelake(from, from_type, n, to, to_type);
+#elif NK_TARGET_SKYLAKE
+    nk_cast_skylake(from, from_type, n, to, to_type);
+#elif NK_TARGET_HASWELL
+    nk_cast_haswell(from, from_type, n, to, to_type);
+#elif NK_TARGET_RVV
+    nk_cast_rvv(from, from_type, n, to, to_type);
+#elif NK_TARGET_NEON
+    nk_cast_neon(from, from_type, n, to, to_type);
+#else
+    nk_cast_serial(from, from_type, n, to, to_type);
+#endif
+}
+
+NK_PUBLIC void nk_f16_to_f32(nk_f16_t const *src, nk_f32_t *dest) {
+#if NK_TARGET_SAPPHIRE
+    nk_f16_to_f32_sapphire(src, dest);
+#elif NK_TARGET_HASWELL
+    nk_f16_to_f32_haswell(src, dest);
+#elif NK_TARGET_NEON
+    nk_f16_to_f32_neon(src, dest);
+#else
+    nk_f16_to_f32_serial(src, dest);
+#endif
+}
+
+NK_PUBLIC void nk_f32_to_f16(nk_f32_t const *src, nk_f16_t *dest) {
+#if NK_TARGET_SAPPHIRE
+    nk_f32_to_f16_sapphire(src, dest);
+#elif NK_TARGET_HASWELL
+    nk_f32_to_f16_haswell(src, dest);
+#elif NK_TARGET_NEON
+    nk_f32_to_f16_neon(src, dest);
+#else
+    nk_f32_to_f16_serial(src, dest);
+#endif
+}
+
+NK_PUBLIC void nk_bf16_to_f32(nk_bf16_t const *src, nk_f32_t *dest) { nk_bf16_to_f32_serial(src, dest); }
+NK_PUBLIC void nk_f32_to_bf16(nk_f32_t const *src, nk_bf16_t *dest) { nk_f32_to_bf16_serial(src, dest); }
+NK_PUBLIC void nk_e4m3_to_f32(nk_e4m3_t const *src, nk_f32_t *dest) { nk_e4m3_to_f32_serial(src, dest); }
+NK_PUBLIC void nk_f32_to_e4m3(nk_f32_t const *src, nk_e4m3_t *dest) { nk_f32_to_e4m3_serial(src, dest); }
+NK_PUBLIC void nk_e5m2_to_f32(nk_e5m2_t const *src, nk_f32_t *dest) { nk_e5m2_to_f32_serial(src, dest); }
+NK_PUBLIC void nk_f32_to_e5m2(nk_f32_t const *src, nk_e5m2_t *dest) { nk_f32_to_e5m2_serial(src, dest); }
+NK_PUBLIC void nk_e2m3_to_f32(nk_e2m3_t const *src, nk_f32_t *dest) { nk_e2m3_to_f32_serial(src, dest); }
+NK_PUBLIC void nk_f32_to_e2m3(nk_f32_t const *src, nk_e2m3_t *dest) { nk_f32_to_e2m3_serial(src, dest); }
+NK_PUBLIC void nk_e3m2_to_f32(nk_e3m2_t const *src, nk_f32_t *dest) { nk_e3m2_to_f32_serial(src, dest); }
+NK_PUBLIC void nk_f32_to_e3m2(nk_f32_t const *src, nk_e3m2_t *dest) { nk_f32_to_e3m2_serial(src, dest); }
+
+#endif // !NK_DYNAMIC_DISPATCH
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_CAST_H

package/include/numkong/curved/README.md

@@ -0,0 +1,223 @@
+# Curved Space Distances in NumKong
+
+NumKong implements distance functions for curved metric spaces: bilinear forms compute $a^T C b$ for an arbitrary metric tensor $C$, while Mahalanobis distance generalizes Euclidean distance to account for correlations between dimensions.
+Complex bilinear forms extend this to Hermitian inner products.
+These operations are central to Gaussian process inference, metric learning, and statistical distance measures.
+
+The bilinear form for real vectors is:
+
+```math
+\text{bilinear}(a, b, C) = a^T C b = \sum_{i=0}^{n-1} \sum_{j=0}^{n-1} a_i \cdot c_{ij} \cdot b_j
+```
+
+The Mahalanobis distance is:
+
+```math
+\text{mahalanobis}(a, b, C) = \sqrt{(a - b)^T C (a - b)}
+```
+
+For complex vectors, the bilinear form uses the conjugate transpose:
+
+```math
+\text{bilinear}(a, b, C) = a^H C b = \sum_{i=0}^{n-1} \sum_{j=0}^{n-1} \bar{a_i} \cdot c_{ij} \cdot b_j
+```
+
+Reformulating as Python pseudocode:
+
+```python
+import numpy as np
+
+def bilinear(a: np.ndarray, b: np.ndarray, C: np.ndarray) -> float:
+    return a @ C @ b
+
+def mahalanobis(a: np.ndarray, b: np.ndarray, C: np.ndarray) -> float:
+    diff = a - b
+    return np.sqrt(diff @ C @ diff)
+
+def bilinear_complex(a: np.ndarray, b: np.ndarray, C: np.ndarray) -> complex:
+    return np.conj(a) @ C @ b
+```
+
+## Input & Output Types
+
+Real bilinear and Mahalanobis:
+
+| Input Type | Output Type | Description                                    |
+| ---------- | ----------- | ---------------------------------------------- |
+| `f64`      | `f64`       | 64-bit IEEE 754 double precision               |
+| `f32`      | `f32`       | 32-bit IEEE 754 single precision               |
+| `f16`      | `f32`       | 16-bit IEEE 754 half precision, widened output |
+| `bf16`     | `f32`       | 16-bit brain float, widened output             |
+
+Complex bilinear:
+
+| Input Type | Output Type | Description                                |
+| ---------- | ----------- | ------------------------------------------ |
+| `f64c`     | `f64c`      | 64-bit complex pairs                       |
+| `f32c`     | `f32c`      | 32-bit complex pairs                       |
+| `f16c`     | `f32c`      | 16-bit complex pairs, widened output       |
+| `bf16c`    | `f32c`      | 16-bit brain complex pairs, widened output |
+
+## Optimizations
+
+### Row-Major Streaming with Nested Dot2
+
+`nk_bilinear_f64_skylake`, `nk_mahalanobis_f64_skylake` decompose the bilinear form $a^T C b$ as $\sum_i a_i \cdot \text{dot}(C_i, b)$ where $C_i$ is the $i$-th row of the metric tensor.
+Each inner dot product uses Dot2 compensation — TwoProd via FMA captures the rounding error of each $c_{ij} \cdot b_j$ product exactly, and a TwoSum chain propagates it through the accumulator.
+The outer sum over rows uses a second level of compensation, tracking the rounding error of each $a_i \cdot r_i$ accumulation.
+This nested structure gives $O(n)$ cache-friendly sequential access to the $n \times n$ matrix $C$, since each row is read once and discarded.
+`nk_bilinear_f32_neon`, `nk_bilinear_f32_skylake`, `nk_mahalanobis_f32_neon`, `nk_mahalanobis_f32_skylake` use the same row-major streaming pattern but accumulate in `f64` instead of Dot2, which provides sufficient precision for `f32` inputs.
+
+### SME Outer-Product Accumulation
+
+`nk_bilinear_f32_smef64`, `nk_bilinear_f64_smef64`, `nk_bilinear_f32c_smef64`, `nk_bilinear_f64c_smef64`, `nk_mahalanobis_f32_smef64`, `nk_mahalanobis_f64_smef64` use the Scalable Matrix Extension to compute the bilinear form as an outer-product accumulation.
+Each `FMOPA` instruction performs a rank-1 update $a_i \cdot b^T$ into the SME ZA tile array, and the matrix $C$ is streamed row-by-row and multiplied into the accumulator.
+This is fundamentally different from the row-major dot approach — it reformulates $a^T C b$ as a matrix-multiply problem where SME's 2D tile registers can exploit the matrix engine's throughput.
+For dimensions that align to the tile size, this approach achieves near-peak throughput; dimensions that do not align fall back to NEON for cleanup of the residual elements.
+
+### Complex Bilinear Decomposition
+
+`nk_bilinear_f32c_neon`, `nk_bilinear_f32c_skylake`, `nk_bilinear_f64c_skylake` compute $a^H C b$ where each element involves 4 real multiplications from the complex product $\bar{a_i} \cdot c_{ij} \cdot b_j$.
+The kernel decomposes this into real and imaginary dot products over rows of $C$: for each row $i$, it computes the real part as $a_{i,re} \cdot \text{dot}(C_i, b)_{re} + a_{i,im} \cdot \text{dot}(C_i, b)_{im}$ and the imaginary part with the conjugation baked in as sign flips.
+This fuses the conjugation of $a$ into the sign of the cross terms rather than explicitly negating the imaginary components, saving one negate operation per element.
+
+## Performance
+
+The following performance tables are produced by manually re-running `nk_test` and `nk_bench` included internal tools to measure both accuracy and throughput at different input shapes.
+The input size is controlled by the `NK_CURVED_DIMENSIONS` environment variable.
+The metric tensor is a square matrix of side $N$, so each bilinear form $\mathbf{x}^\top M \mathbf{x}$ has $O(N^2)$ arithmetic complexity.
+Columns show matrix side length: 256², 1024², 4096².
+The throughput is measured in GSO/s as Giga Scalar Operations per Second.
+Accuracy is reported as mean ULP (units in last place) averaged over all test pairs — the average number of representable floating-point values between the computed result and the exact answer.
+Each kernel runs for at least 20 seconds per configuration.
+Benchmark threads are pinned to specific cores; on machines with heterogeneous core types (e.g., Apple P/E cores), only the fastest cores are used.
+Workloads that significantly degrade CPU frequencies (Intel AMX, Apple SME) run in separate passes to avoid affecting throughput measurements of other kernels.
+Rows marked `🧩` use external BLAS baselines rather than NumKong kernels.
+
+### Intel Sapphire Rapids
+
+#### Native
+
+| Kernel                        |                     256² |                    1024² |                    4096² |
+| :---------------------------- | -----------------------: | -----------------------: | -----------------------: |
+| __f64c__                      | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `bilinear_f64c_with_blas` 🧩   |               1.25 gso/s |               1.36 gso/s |               1.38 gso/s |
+| `nk_bilinear_f64c_serial`     |    0.0862 gso/s, 0.5 ulp |     0.161 gso/s, 0.2 ulp |     0.171 gso/s, 0.5 ulp |
+| `nk_bilinear_f64c_skylake`    |     0.583 gso/s, 3.5 ulp |     0.718 gso/s, 3.5 ulp |     0.765 gso/s, 3.5 ulp |
+| __f32c__                      | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `bilinear_f32c_with_blas` 🧩   |               2.14 gso/s |               2.61 gso/s |               2.57 gso/s |
+| `nk_bilinear_f32c_serial`     |       0.756 gso/s, 0 ulp |        1.37 gso/s, 0 ulp |        1.37 gso/s, 0 ulp |
+| `nk_bilinear_f32c_skylake`    |        1.72 gso/s, 0 ulp |        1.75 gso/s, 0 ulp |        1.46 gso/s, 0 ulp |
+| __bf16c__                     | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_bf16c_serial`    |       0.154 gso/s, 5 ulp |     0.158 gso/s, 5.8 ulp |       0.155 gso/s, 5 ulp |
+| `nk_bilinear_bf16c_genoa`     |        2.81 gso/s, 5 ulp |        4.57 gso/s, 5 ulp |        4.47 gso/s, 5 ulp |
+| __f16c__                      | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f16c_serial`     |     0.585 gso/s, 7.2 ulp |     0.592 gso/s, 7.2 ulp |     0.600 gso/s, 7.2 ulp |
+| __f64__                       | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `bilinear_f64_with_blas` 🧩    |               2.84 gso/s |               3.23 gso/s |               3.14 gso/s |
+| `nk_bilinear_f64_serial`      |     0.291 gso/s, 0.7 ulp |     0.565 gso/s, 0.4 ulp |     0.577 gso/s, 0.7 ulp |
+| `nk_mahalanobis_f64_serial`   |       0.267 gso/s, 0 ulp |       0.537 gso/s, 0 ulp |       0.539 gso/s, 0 ulp |
+| `nk_bilinear_f64_skylake`     |      1.79 gso/s, 1.6 ulp |      1.71 gso/s, 1.3 ulp |        1.59 gso/s, 1 ulp |
+| `nk_mahalanobis_f64_skylake`  |        1.77 gso/s, 0 ulp |        1.82 gso/s, 0 ulp |      2.12 gso/s, 0.2 ulp |
+| __f32__                       | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `bilinear_f32_with_blas` 🧩    |               4.09 gso/s |               5.61 gso/s |               6.59 gso/s |
+| `nk_bilinear_f32_serial`      |        1.19 gso/s, 0 ulp |        2.71 gso/s, 0 ulp |        2.68 gso/s, 0 ulp |
+| `nk_mahalanobis_f32_serial`   |        2.36 gso/s, 0 ulp |        2.53 gso/s, 0 ulp |        2.40 gso/s, 0 ulp |
+| `nk_bilinear_f32_haswell`     |        3.45 gso/s, 0 ulp |        3.66 gso/s, 0 ulp |        3.24 gso/s, 0 ulp |
+| `nk_mahalanobis_f32_haswell`  |        3.37 gso/s, 0 ulp |        3.28 gso/s, 0 ulp |        3.30 gso/s, 0 ulp |
+| `nk_bilinear_f32_skylake`     |        3.68 gso/s, 0 ulp |        3.08 gso/s, 0 ulp |        2.71 gso/s, 0 ulp |
+| `nk_mahalanobis_f32_skylake`  |        3.45 gso/s, 0 ulp |        2.94 gso/s, 0 ulp |        3.32 gso/s, 0 ulp |
+| __bf16__                      | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_bf16_serial`     |      0.321 gso/s, 16 ulp |      0.331 gso/s, 13 ulp |      0.314 gso/s, 12 ulp |
+| `nk_mahalanobis_bf16_serial`  |     0.216 gso/s, 2.2 ulp |     0.215 gso/s, 2.1 ulp |     0.211 gso/s, 2.3 ulp |
+| `nk_bilinear_bf16_haswell`    |       6.75 gso/s, 11 ulp |       7.04 gso/s, 13 ulp |       6.80 gso/s, 13 ulp |
+| `nk_mahalanobis_bf16_haswell` |        5.93 gso/s, 1 ulp |        5.77 gso/s, 1 ulp |        5.86 gso/s, 1 ulp |
+| `nk_bilinear_bf16_genoa`      |       6.22 gso/s, 18 ulp |       10.9 gso/s, 18 ulp |       10.3 gso/s, 18 ulp |
+| `nk_mahalanobis_bf16_genoa`   |    7.04 gso/s, 8.55K ulp |    8.76 gso/s, 8.41K ulp |    8.57 gso/s, 8.41K ulp |
+| __f16__                       | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f16_serial`      |      0.654 gso/s, 23 ulp |      0.652 gso/s, 23 ulp |      0.657 gso/s, 23 ulp |
+| `nk_mahalanobis_f16_serial`   |     0.510 gso/s, 2.7 ulp |     0.520 gso/s, 3.2 ulp |     0.500 gso/s, 2.7 ulp |
+| `nk_bilinear_f16_haswell`     |       7.36 gso/s, 37 ulp |       7.30 gso/s, 37 ulp |       7.29 gso/s, 37 ulp |
+| `nk_mahalanobis_f16_haswell`  |        6.75 gso/s, 1 ulp |        6.24 gso/s, 1 ulp |        6.83 gso/s, 1 ulp |
+
+#### WASM
+
+Measured with Wasmtime v42 (Cranelift backend).
+
+| Kernel                     |                     256² |                    1024² |                    4096² |
+| :------------------------- | -----------------------: | -----------------------: | -----------------------: |
+| __f64c__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f64c_serial`  |      0.21 gso/s, 1.2 ulp |      0.21 gso/s, 1.2 ulp |      0.21 gso/s, 1.2 ulp |
+| __f32c__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f32c_serial`  |        1.10 gso/s, 0 ulp |        1.07 gso/s, 0 ulp |        1.10 gso/s, 0 ulp |
+| __bf16c__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_bf16c_serial` |      1.26 gso/s, 9.8 ulp |      1.31 gso/s, 9.8 ulp |      1.27 gso/s, 9.5 ulp |
+| __f16c__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f16c_serial`  |       0.40 gso/s, 39 ulp |       0.38 gso/s, 39 ulp |       0.40 gso/s, 39 ulp |
+| __f64__                    | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f64_serial`   |      0.49 gso/s, 0.6 ulp |      0.49 gso/s, 0.6 ulp |      0.48 gso/s, 0.6 ulp |
+| __f32__                    | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f32_serial`   |        2.54 gso/s, 0 ulp |        2.62 gso/s, 0 ulp |        2.53 gso/s, 0 ulp |
+| __bf16__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_bf16_serial`  |       2.91 gso/s, 27 ulp |       2.90 gso/s, 22 ulp |       2.98 gso/s, 22 ulp |
+| __f16__                    | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f16_serial`   |       0.76 gso/s, 74 ulp |       0.76 gso/s, 74 ulp |       0.78 gso/s, 74 ulp |
+
+### Apple M4
+
+#### Native
+
+| Kernel                          |                     256² |                    1024² |                    4096² |
+| :------------------------------ | -----------------------: | -----------------------: | -----------------------: |
+| __f64c__                        | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f64c_serial`       |     0.368 gso/s, 2.2 ulp |     0.371 gso/s, 2.2 ulp |     0.367 gso/s, 2.2 ulp |
+| __f32c__                        | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f32c_serial`       |        2.33 gso/s, 0 ulp |        2.27 gso/s, 0 ulp |        2.28 gso/s, 0 ulp |
+| `nk_bilinear_f32c_neon`         |        2.11 gso/s, 0 ulp |        1.89 gso/s, 0 ulp |        1.85 gso/s, 0 ulp |
+| __bf16c__                       | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_bf16c_serial`      |     2.83 gso/s, 33.0 ulp |     2.54 gso/s, 34.5 ulp |     2.49 gso/s, 34.5 ulp |
+| `nk_bilinear_bf16c_neonbfdot`   |     5.05 gso/s, 17.0 ulp |     4.20 gso/s, 17.0 ulp |     4.04 gso/s, 17.0 ulp |
+| __f16c__                        | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f16c_serial`       |     2.81 gso/s, 51.8 ulp |     2.54 gso/s, 51.8 ulp |     2.48 gso/s, 51.8 ulp |
+| `nk_bilinear_f16c_neonhalf`     |     5.00 gso/s, 17.3 ulp |     4.16 gso/s, 17.3 ulp |     4.00 gso/s, 16.4 ulp |
+| __f64__                         | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f64_serial`        |     0.717 gso/s, 0.4 ulp |     0.711 gso/s, 0.4 ulp |     0.721 gso/s, 0.4 ulp |
+| `nk_mahalanobis_f64_serial`     |     0.664 gso/s, 0.5 ulp |     0.667 gso/s, 0.5 ulp |     0.672 gso/s, 0.5 ulp |
+| __f32__                         | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f32_serial`        |        3.92 gso/s, 0 ulp |        3.05 gso/s, 0 ulp |        2.87 gso/s, 0 ulp |
+| `nk_mahalanobis_f32_serial`     |        3.42 gso/s, 0 ulp |        2.88 gso/s, 0 ulp |        2.74 gso/s, 0 ulp |
+| `nk_bilinear_f32_neon`          |        4.90 gso/s, 0 ulp |        3.82 gso/s, 0 ulp |        3.49 gso/s, 0 ulp |
+| `nk_mahalanobis_f32_neon`       |        4.68 gso/s, 0 ulp |        3.71 gso/s, 0 ulp |        3.48 gso/s, 0 ulp |
+| __bf16__                        | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_bf16_serial`       |     4.17 gso/s, 20.7 ulp |     3.19 gso/s, 21.2 ulp |     2.94 gso/s, 20.7 ulp |
+| `nk_mahalanobis_bf16_serial`    |      3.86 gso/s, 2.1 ulp |      2.98 gso/s, 2.2 ulp |      2.79 gso/s, 2.1 ulp |
+| `nk_bilinear_bf16_neonbfdot`    |     28.0 gso/s, 28.0 ulp |     23.5 gso/s, 41.2 ulp |     20.4 gso/s, 41.1 ulp |
+| `nk_mahalanobis_bf16_neonbfdot` |      9.14 gso/s, 2.2 ulp |      7.93 gso/s, 2.2 ulp |      7.43 gso/s, 2.2 ulp |
+| __f16__                         | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f16_serial`        |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
+| `nk_mahalanobis_f16_serial`     |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
+| `nk_bilinear_f16_neonhalf`      |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
+| `nk_mahalanobis_f16_neonhalf`   |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
+
+#### WASM
+
+Measured with Wasmtime v42 (Cranelift backend).
+
+| Kernel                     |                     256² |                    1024² |                    4096² |
+| :------------------------- | -----------------------: | -----------------------: | -----------------------: |
+| __f64c__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f64c_serial`  |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
+| __f32c__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f32c_serial`  |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
+| __bf16c__                  | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_bf16c_serial` |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
+| __f16c__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f16c_serial`  |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
+| __f64__                    | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f64_serial`   |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
+| __f32__                    | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f32_serial`   |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
+| __bf16__                   | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_bf16_serial`  |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |
+| __f16__                    | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
+| `nk_bilinear_f16_serial`   |           ? gso/s, ? ulp |           ? gso/s, ? ulp |           ? gso/s, ? ulp |