numkong 7.0.0 → 7.4.2

package/include/numkong/spatial/svehalf.h

@@ -8,15 +8,15 @@
  *
  *  @section spatial_svehalf_instructions ARM SVE+FP16 Instructions
  *
- *      Intrinsic                   Instruction                     Latency     Throughput
- *      svld1_f16                   LD1H (Z.H, P/Z, [Xn])           4-6cy       2/cy
- *      svsub_f16_x                 FSUB (Z.H, P/M, Z.H, Z.H)       3cy         2/cy
- *      svmla_f16_x                 FMLA (Z.H, P/M, Z.H, Z.H)       4cy         2/cy
- *      svaddv_f16                  FADDV (H, P, Z.H)               6cy         1/cy
- *      svdupq_n_f16                DUP (Z.H, #imm)                 1cy         2/cy
- *      svwhilelt_b16               WHILELT (P.H, Xn, Xm)           2cy         1/cy
- *      svptrue_b16                 PTRUE (P.H, pattern)            1cy         2/cy
- *      svcnth                      CNTH (Xd)                       1cy         2/cy
+ *      Intrinsic      Instruction                V1
+ *      svld1_f16      LD1H (Z.H, P/Z, [Xn])      4-6cy @ 2p
+ *      svsub_f16_x    FSUB (Z.H, P/M, Z.H, Z.H)  3cy @ 2p
+ *      svmla_f16_x    FMLA (Z.H, P/M, Z.H, Z.H)  4cy @ 2p
+ *      svaddv_f16     FADDV (H, P, Z.H)          6cy @ 1p
+ *      svdupq_n_f16   DUP (Z.H, #imm)            1cy @ 2p
+ *      svwhilelt_b16  WHILELT (P.H, Xn, Xm)      2cy @ 1p
+ *      svptrue_b16    PTRUE (P.H, pattern)       1cy @ 2p
+ *      svcnth         CNTH (Xd)                  1cy @ 2p
  *
  *  SVE vector widths vary across implementations: Graviton3 uses 256-bit, while Graviton4/5
  *  and Apple M4+ use 128-bit. Code using svcntb() adapts automatically, but wider vectors
@@ -52,14 +52,27 @@ NK_PUBLIC void nk_sqeuclidean_f16_svehalf(nk_f16_t const *a_enum, nk_f16_t const
     nk_f16_for_arm_simd_t const *a = (nk_f16_for_arm_simd_t const *)(a_enum);
     nk_f16_for_arm_simd_t const *b = (nk_f16_for_arm_simd_t const *)(b_enum);
     do {
-        svbool_t predicate_f32x = svwhilelt_b32_u64(i, n);
-        svfloat16_t a_f16x = svld1_f16(svwhilelt_b16_u64(i, n), a + i);
-        svfloat16_t b_f16x = svld1_f16(svwhilelt_b16_u64(i, n), b + i);
-        svfloat32_t a_f32x = svcvt_f32_f16_x(predicate_f32x, a_f16x);
-        svfloat32_t b_f32x = svcvt_f32_f16_x(predicate_f32x, b_f16x);
-        svfloat32_t diff_f32x = svsub_f32_x(predicate_f32x, a_f32x, b_f32x);
-        d2_f32x = svmla_f32_x(predicate_f32x, d2_f32x, diff_f32x, diff_f32x);
-        i += svcntw();
+        svbool_t predicate_b16x = svwhilelt_b16_u64(i, n);
+        svfloat16_t a_f16x = svld1_f16(predicate_b16x, a + i);
+        svfloat16_t b_f16x = svld1_f16(predicate_b16x, b + i);
+        nk_size_t remaining = n - i < svcnth() ? n - i : svcnth();
+
+        // SVE `svcvt_f32_f16_x` widens only even-indexed f16 elements (0, 2, 4, ...),
+        // so we need two passes: one on the original vector (even elements) and one on
+        // a vector shifted by one position via `svext` (odd elements become even).
+        svbool_t pred_even_b32x = svwhilelt_b32_u64(0u, (remaining + 1) / 2);
+        svfloat32_t a_even_f32x = svcvt_f32_f16_x(pred_even_b32x, a_f16x);
+        svfloat32_t b_even_f32x = svcvt_f32_f16_x(pred_even_b32x, b_f16x);
+        svfloat32_t diff_even_f32x = svsub_f32_x(pred_even_b32x, a_even_f32x, b_even_f32x);
+        d2_f32x = svmla_f32_m(pred_even_b32x, d2_f32x, diff_even_f32x, diff_even_f32x);
+
+        svbool_t pred_odd_b32x = svwhilelt_b32_u64(0u, remaining / 2);
+        svfloat32_t a_odd_f32x = svcvt_f32_f16_x(pred_odd_b32x, svext_f16(a_f16x, a_f16x, 1));
+        svfloat32_t b_odd_f32x = svcvt_f32_f16_x(pred_odd_b32x, svext_f16(b_f16x, b_f16x, 1));
+        svfloat32_t diff_odd_f32x = svsub_f32_x(pred_odd_b32x, a_odd_f32x, b_odd_f32x);
+        d2_f32x = svmla_f32_m(pred_odd_b32x, d2_f32x, diff_odd_f32x, diff_odd_f32x);
+
+        i += svcnth();
     } while (i < n);
     *result = svaddv_f32(svptrue_b32(), d2_f32x);
 }
@@ -77,15 +90,28 @@ NK_PUBLIC void nk_angular_f16_svehalf(nk_f16_t const *a_enum, nk_f16_t const *b_
     nk_f16_for_arm_simd_t const *a = (nk_f16_for_arm_simd_t const *)(a_enum);
     nk_f16_for_arm_simd_t const *b = (nk_f16_for_arm_simd_t const *)(b_enum);
     do {
-        svbool_t predicate_f32x = svwhilelt_b32_u64(i, n);
-        svfloat16_t a_f16x = svld1_f16(svwhilelt_b16_u64(i, n), a + i);
-        svfloat16_t b_f16x = svld1_f16(svwhilelt_b16_u64(i, n), b + i);
-        svfloat32_t a_f32x = svcvt_f32_f16_x(predicate_f32x, a_f16x);
-        svfloat32_t b_f32x = svcvt_f32_f16_x(predicate_f32x, b_f16x);
-        ab_f32x = svmla_f32_x(predicate_f32x, ab_f32x, a_f32x, b_f32x);
-        a2_f32x = svmla_f32_x(predicate_f32x, a2_f32x, a_f32x, a_f32x);
-        b2_f32x = svmla_f32_x(predicate_f32x, b2_f32x, b_f32x, b_f32x);
-        i += svcntw();
+        svbool_t predicate_b16x = svwhilelt_b16_u64(i, n);
+        svfloat16_t a_f16x = svld1_f16(predicate_b16x, a + i);
+        svfloat16_t b_f16x = svld1_f16(predicate_b16x, b + i);
+        nk_size_t remaining = n - i < svcnth() ? n - i : svcnth();
+
+        // Even-indexed f16 elements (0, 2, 4, ...)
+        svbool_t pred_even_b32x = svwhilelt_b32_u64(0u, (remaining + 1) / 2);
+        svfloat32_t a_even_f32x = svcvt_f32_f16_x(pred_even_b32x, a_f16x);
+        svfloat32_t b_even_f32x = svcvt_f32_f16_x(pred_even_b32x, b_f16x);
+        ab_f32x = svmla_f32_m(pred_even_b32x, ab_f32x, a_even_f32x, b_even_f32x);
+        a2_f32x = svmla_f32_m(pred_even_b32x, a2_f32x, a_even_f32x, a_even_f32x);
+        b2_f32x = svmla_f32_m(pred_even_b32x, b2_f32x, b_even_f32x, b_even_f32x);
+
+        // Odd-indexed f16 elements (1, 3, 5, ...) via svext shift-by-1
+        svbool_t pred_odd_b32x = svwhilelt_b32_u64(0u, remaining / 2);
+        svfloat32_t a_odd_f32x = svcvt_f32_f16_x(pred_odd_b32x, svext_f16(a_f16x, a_f16x, 1));
+        svfloat32_t b_odd_f32x = svcvt_f32_f16_x(pred_odd_b32x, svext_f16(b_f16x, b_f16x, 1));
+        ab_f32x = svmla_f32_m(pred_odd_b32x, ab_f32x, a_odd_f32x, b_odd_f32x);
+        a2_f32x = svmla_f32_m(pred_odd_b32x, a2_f32x, a_odd_f32x, a_odd_f32x);
+        b2_f32x = svmla_f32_m(pred_odd_b32x, b2_f32x, b_odd_f32x, b_odd_f32x);
+
+        i += svcnth();
     } while (i < n);
 
     nk_f32_t ab_f32 = svaddv_f32(svptrue_b32(), ab_f32x);

package/include/numkong/spatial/svesdot.h

@@ -0,0 +1,142 @@
+/**
+ *  @brief SIMD-accelerated Spatial Similarity Measures for SVE SDOT.
+ *  @file include/numkong/spatial/svesdot.h
+ *  @author Ash Vardanian
+ *  @date April 3, 2026
+ *
+ *  @sa include/numkong/spatial.h
+ *
+ *  @section spatial_svesdot_instructions ARM SVE+DotProd Instructions
+ *
+ *      Intrinsic      Instruction              V1
+ *      svld1_s8       LD1B (Z.B, P/Z, [Xn])    4-6cy @ 2p
+ *      svld1_u8       LD1B (Z.B, P/Z, [Xn])    4-6cy @ 2p
+ *      svdot_s32      SDOT (Z.S, Z.B, Z.B)     3cy @ 2p
+ *      svdot_u32      UDOT (Z.S, Z.B, Z.B)     3cy @ 2p
+ *      svabd_s8_x     SABD (Z.B, P/M, Z.B)     3cy @ 2p
+ *      svabd_u8_x     UABD (Z.B, P/M, Z.B)     3cy @ 2p
+ *      svaddv_s32     SADDV (D, P, Z.S)        6cy @ 1p
+ *      svaddv_u32     UADDV (D, P, Z.S)        6cy @ 1p
+ *      svwhilelt_b8   WHILELT (P.B, Xn, Xm)    2cy @ 1p
+ *      svcntb         CNTB (Xd)                1cy @ 2p
+ *
+ *  SVE vector widths vary across implementations: Graviton3 uses 256-bit, while Graviton4/5
+ *  and Apple M4+ use 128-bit. Code using svcntb() adapts automatically, but wider vectors
+ *  process more elements per iteration with identical latencies.
+ *
+ *  For L2 distance, SABD/UABD computes |a-b| per byte, then UDOT squares and accumulates.
+ *  Angular distance uses SDOT/UDOT directly for dot product and norm computations.
+ */
+#ifndef NK_SPATIAL_SVESDOT_H
+#define NK_SPATIAL_SVESDOT_H
+
+#if NK_TARGET_ARM_
+#if NK_TARGET_SVESDOT
+
+#include "numkong/types.h"
+#include "numkong/spatial/neon.h" // `nk_angular_normalize_f32_neon_`, `nk_f32_sqrt_neon`
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+sve+dotprod"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=armv8.2-a+sve+dotprod")
+#endif
+
+NK_PUBLIC void nk_sqeuclidean_i8_svesdot(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_u32_t *result) {
+    nk_size_t i = 0;
+    svuint32_t distance_sq_u32x = svdup_u32(0);
+    do {
+        svbool_t predicate_b8x = svwhilelt_b8_u64(i, n);
+        svint8_t a_i8x = svld1_s8(predicate_b8x, a + i);
+        svint8_t b_i8x = svld1_s8(predicate_b8x, b + i);
+        svuint8_t diff_u8x = svreinterpret_u8_s8(svabd_s8_x(predicate_b8x, a_i8x, b_i8x));
+        distance_sq_u32x = svdot_u32(distance_sq_u32x, diff_u8x, diff_u8x);
+        i += svcntb();
+    } while (i < n);
+    *result = (nk_u32_t)svaddv_u32(svptrue_b32(), distance_sq_u32x);
+}
+NK_PUBLIC void nk_euclidean_i8_svesdot(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_u32_t distance_sq_u32;
+    nk_sqeuclidean_i8_svesdot(a, b, n, &distance_sq_u32);
+    *result = nk_f32_sqrt_neon((nk_f32_t)distance_sq_u32);
+}
+
+NK_PUBLIC void nk_angular_i8_svesdot(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_size_t i = 0;
+    svint32_t ab_i32x = svdup_s32(0);
+    svint32_t a2_i32x = svdup_s32(0);
+    svint32_t b2_i32x = svdup_s32(0);
+    do {
+        svbool_t predicate_b8x = svwhilelt_b8_u64(i, n);
+        svint8_t a_i8x = svld1_s8(predicate_b8x, a + i);
+        svint8_t b_i8x = svld1_s8(predicate_b8x, b + i);
+        ab_i32x = svdot_s32(ab_i32x, a_i8x, b_i8x);
+        a2_i32x = svdot_s32(a2_i32x, a_i8x, a_i8x);
+        b2_i32x = svdot_s32(b2_i32x, b_i8x, b_i8x);
+        i += svcntb();
+    } while (i < n);
+
+    nk_i32_t ab = (nk_i32_t)svaddv_s32(svptrue_b32(), ab_i32x);
+    nk_i32_t a2 = (nk_i32_t)svaddv_s32(svptrue_b32(), a2_i32x);
+    nk_i32_t b2 = (nk_i32_t)svaddv_s32(svptrue_b32(), b2_i32x);
+    *result = nk_angular_normalize_f32_neon_((nk_f32_t)ab, (nk_f32_t)a2, (nk_f32_t)b2);
+}
+
+NK_PUBLIC void nk_sqeuclidean_u8_svesdot(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_u32_t *result) {
+    nk_size_t i = 0;
+    svuint32_t distance_sq_u32x = svdup_u32(0);
+    do {
+        svbool_t predicate_b8x = svwhilelt_b8_u64(i, n);
+        svuint8_t a_u8x = svld1_u8(predicate_b8x, a + i);
+        svuint8_t b_u8x = svld1_u8(predicate_b8x, b + i);
+        svuint8_t diff_u8x = svabd_u8_x(predicate_b8x, a_u8x, b_u8x);
+        distance_sq_u32x = svdot_u32(distance_sq_u32x, diff_u8x, diff_u8x);
+        i += svcntb();
+    } while (i < n);
+    *result = (nk_u32_t)svaddv_u32(svptrue_b32(), distance_sq_u32x);
+}
+NK_PUBLIC void nk_euclidean_u8_svesdot(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_u32_t distance_sq_u32;
+    nk_sqeuclidean_u8_svesdot(a, b, n, &distance_sq_u32);
+    *result = nk_f32_sqrt_neon((nk_f32_t)distance_sq_u32);
+}
+
+NK_PUBLIC void nk_angular_u8_svesdot(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_size_t i = 0;
+    svuint32_t ab_u32x = svdup_u32(0);
+    svuint32_t a2_u32x = svdup_u32(0);
+    svuint32_t b2_u32x = svdup_u32(0);
+    do {
+        svbool_t predicate_b8x = svwhilelt_b8_u64(i, n);
+        svuint8_t a_u8x = svld1_u8(predicate_b8x, a + i);
+        svuint8_t b_u8x = svld1_u8(predicate_b8x, b + i);
+        ab_u32x = svdot_u32(ab_u32x, a_u8x, b_u8x);
+        a2_u32x = svdot_u32(a2_u32x, a_u8x, a_u8x);
+        b2_u32x = svdot_u32(b2_u32x, b_u8x, b_u8x);
+        i += svcntb();
+    } while (i < n);
+
+    nk_u32_t ab = (nk_u32_t)svaddv_u32(svptrue_b32(), ab_u32x);
+    nk_u32_t a2 = (nk_u32_t)svaddv_u32(svptrue_b32(), a2_u32x);
+    nk_u32_t b2 = (nk_u32_t)svaddv_u32(svptrue_b32(), b2_u32x);
+    *result = nk_angular_normalize_f32_neon_((nk_f32_t)ab, (nk_f32_t)a2, (nk_f32_t)b2);
+}
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_SVESDOT
+#endif // NK_TARGET_ARM_
+#endif // NK_SPATIAL_SVESDOT_H

package/include/numkong/spatial/v128relaxed.h

@@ -64,7 +64,7 @@ NK_INTERNAL nk_f64_t nk_angular_normalize_f64_v128relaxed_(nk_f64_t ab, nk_f64_t
     return result > 0.0 ? result : 0.0;
 }
 
-#pragma region - Traditional Floats
+#pragma region F32 and F64 Floats
 
 NK_PUBLIC void nk_sqeuclidean_f32_v128relaxed(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f64_t *result) {
     v128_t sum_f64x2 = wasm_f64x2_splat(0.0);
@@ -83,8 +83,8 @@ nk_sqeuclidean_f32_v128relaxed_cycle:
         nk_load_b64_serial_(b_scalars, &b_f32_vec);
         a_scalars += 2, b_scalars += 2, count_scalars -= 2;
     }
-    v128_t a_f32x2 = wasm_v128_load64_zero(&a_f32_vec.u64);
-    v128_t b_f32x2 = wasm_v128_load64_zero(&b_f32_vec.u64);
+    v128_t a_f32x2 = wasm_i64x2_splat(a_f32_vec.u64);
+    v128_t b_f32x2 = wasm_i64x2_splat(b_f32_vec.u64);
     v128_t a_f64x2 = wasm_f64x2_promote_low_f32x4(a_f32x2);
     v128_t b_f64x2 = wasm_f64x2_promote_low_f32x4(b_f32x2);
     v128_t diff_f64x2 = wasm_f64x2_sub(a_f64x2, b_f64x2);
@@ -152,8 +152,8 @@ nk_angular_f32_v128relaxed_cycle:
     }
 
     // Upcast F32x2 → F64x2 for high-precision accumulation
-    v128_t a_f32x2 = wasm_v128_load64_zero(&a_f32_vec.u64);
-    v128_t b_f32x2 = wasm_v128_load64_zero(&b_f32_vec.u64);
+    v128_t a_f32x2 = wasm_i64x2_splat(a_f32_vec.u64);
+    v128_t b_f32x2 = wasm_i64x2_splat(b_f32_vec.u64);
     v128_t a_f64x2 = wasm_f64x2_promote_low_f32x4(a_f32x2);
     v128_t b_f64x2 = wasm_f64x2_promote_low_f32x4(b_f32x2);
 
@@ -203,8 +203,8 @@ nk_angular_f64_v128relaxed_cycle:
     *result = nk_angular_normalize_f64_v128relaxed_(ab, a2, b2);
 }
 
-#pragma endregion - Traditional Floats
-#pragma region - Smaller Floats
+#pragma endregion F32 and F64 Floats
+#pragma region F16 and BF16 Floats
 
 NK_PUBLIC void nk_sqeuclidean_f16_v128relaxed(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *result) {
     v128_t sum_f32x4 = wasm_f32x4_splat(0.0f);
@@ -286,31 +286,30 @@ nk_angular_f16_v128relaxed_cycle:
 
 NK_PUBLIC void nk_sqeuclidean_bf16_v128relaxed(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_f32_t *result) {
     v128_t sum_f32x4 = wasm_f32x4_splat(0.0f);
+    v128_t mask_high_u32x4 = wasm_i32x4_splat((int)0xFFFF0000);
     nk_bf16_t const *a_scalars = a, *b_scalars = b;
     nk_size_t count_scalars = n;
-    nk_b64_vec_t a_bf16_vec, b_bf16_vec;
+    nk_b128_vec_t a_bf16_vec, b_bf16_vec;
 
 nk_sqeuclidean_bf16_v128relaxed_cycle:
-    // Tail or full load
-    if (count_scalars < 4) {
-        nk_partial_load_b16x4_serial_(a_scalars, &a_bf16_vec, count_scalars);
-        nk_partial_load_b16x4_serial_(b_scalars, &b_bf16_vec, count_scalars);
+    if (count_scalars < 8) {
+        nk_partial_load_b16x8_serial_(a_scalars, &a_bf16_vec, count_scalars);
+        nk_partial_load_b16x8_serial_(b_scalars, &b_bf16_vec, count_scalars);
         count_scalars = 0;
     }
     else {
-        nk_load_b64_serial_(a_scalars, &a_bf16_vec);
-        nk_load_b64_serial_(b_scalars, &b_bf16_vec);
-        a_scalars += 4, b_scalars += 4, count_scalars -= 4;
+        nk_load_b128_v128relaxed_(a_scalars, &a_bf16_vec);
+        nk_load_b128_v128relaxed_(b_scalars, &b_bf16_vec);
+        a_scalars += 8, b_scalars += 8, count_scalars -= 8;
     }
-
-    // Convert bf16 → f32 (4 elements)
-    nk_b128_vec_t a_f32_vec = nk_bf16x4_to_f32x4_v128relaxed_(a_bf16_vec);
-    nk_b128_vec_t b_f32_vec = nk_bf16x4_to_f32x4_v128relaxed_(b_bf16_vec);
-
-    // Accumulate (a - b)²
-    v128_t diff_f32x4 = wasm_f32x4_sub(a_f32_vec.v128, b_f32_vec.v128);
-    sum_f32x4 = wasm_f32x4_relaxed_madd(diff_f32x4, diff_f32x4, sum_f32x4);
-
+    v128_t a_even_f32x4 = wasm_i32x4_shl(a_bf16_vec.v128, 16);
+    v128_t b_even_f32x4 = wasm_i32x4_shl(b_bf16_vec.v128, 16);
+    v128_t diff_even_f32x4 = wasm_f32x4_sub(a_even_f32x4, b_even_f32x4);
+    sum_f32x4 = wasm_f32x4_relaxed_madd(diff_even_f32x4, diff_even_f32x4, sum_f32x4);
+    v128_t a_odd_f32x4 = wasm_v128_and(a_bf16_vec.v128, mask_high_u32x4);
+    v128_t b_odd_f32x4 = wasm_v128_and(b_bf16_vec.v128, mask_high_u32x4);
+    v128_t diff_odd_f32x4 = wasm_f32x4_sub(a_odd_f32x4, b_odd_f32x4);
+    sum_f32x4 = wasm_f32x4_relaxed_madd(diff_odd_f32x4, diff_odd_f32x4, sum_f32x4);
     if (count_scalars) goto nk_sqeuclidean_bf16_v128relaxed_cycle;
 
     *result = nk_reduce_add_f32x4_v128relaxed_(sum_f32x4);
@@ -326,44 +325,297 @@ NK_PUBLIC void nk_angular_bf16_v128relaxed(nk_bf16_t const *a, nk_bf16_t const *
     v128_t ab_f32x4 = wasm_f32x4_splat(0.0f);
     v128_t a2_f32x4 = wasm_f32x4_splat(0.0f);
     v128_t b2_f32x4 = wasm_f32x4_splat(0.0f);
+    v128_t mask_high_u32x4 = wasm_i32x4_splat((int)0xFFFF0000);
     nk_bf16_t const *a_scalars = a, *b_scalars = b;
     nk_size_t count_scalars = n;
-    nk_b64_vec_t a_bf16_vec, b_bf16_vec;
+    nk_b128_vec_t a_bf16_vec, b_bf16_vec;
 
 nk_angular_bf16_v128relaxed_cycle:
+    if (count_scalars < 8) {
+        nk_partial_load_b16x8_serial_(a_scalars, &a_bf16_vec, count_scalars);
+        nk_partial_load_b16x8_serial_(b_scalars, &b_bf16_vec, count_scalars);
+        count_scalars = 0;
+    }
+    else {
+        nk_load_b128_v128relaxed_(a_scalars, &a_bf16_vec);
+        nk_load_b128_v128relaxed_(b_scalars, &b_bf16_vec);
+        a_scalars += 8, b_scalars += 8, count_scalars -= 8;
+    }
+    v128_t a_even_f32x4 = wasm_i32x4_shl(a_bf16_vec.v128, 16);
+    v128_t b_even_f32x4 = wasm_i32x4_shl(b_bf16_vec.v128, 16);
+    ab_f32x4 = wasm_f32x4_relaxed_madd(a_even_f32x4, b_even_f32x4, ab_f32x4);
+    a2_f32x4 = wasm_f32x4_relaxed_madd(a_even_f32x4, a_even_f32x4, a2_f32x4);
+    b2_f32x4 = wasm_f32x4_relaxed_madd(b_even_f32x4, b_even_f32x4, b2_f32x4);
+    v128_t a_odd_f32x4 = wasm_v128_and(a_bf16_vec.v128, mask_high_u32x4);
+    v128_t b_odd_f32x4 = wasm_v128_and(b_bf16_vec.v128, mask_high_u32x4);
+    ab_f32x4 = wasm_f32x4_relaxed_madd(a_odd_f32x4, b_odd_f32x4, ab_f32x4);
+    a2_f32x4 = wasm_f32x4_relaxed_madd(a_odd_f32x4, a_odd_f32x4, a2_f32x4);
+    b2_f32x4 = wasm_f32x4_relaxed_madd(b_odd_f32x4, b_odd_f32x4, b2_f32x4);
+    if (count_scalars) goto nk_angular_bf16_v128relaxed_cycle;
+
+    nk_f32_t ab = nk_reduce_add_f32x4_v128relaxed_(ab_f32x4);
+    nk_f32_t a2 = nk_reduce_add_f32x4_v128relaxed_(a2_f32x4);
+    nk_f32_t b2 = nk_reduce_add_f32x4_v128relaxed_(b2_f32x4);
+    *result = (nk_f32_t)nk_angular_normalize_f64_v128relaxed_((nk_f64_t)ab, (nk_f64_t)a2, (nk_f64_t)b2);
+}
+
+#pragma endregion F16 and BF16 Floats
+#pragma region FP8 Floats
+
+NK_PUBLIC void nk_sqeuclidean_e4m3_v128relaxed(nk_e4m3_t const *a, nk_e4m3_t const *b, nk_size_t n, nk_f32_t *result) {
+    v128_t sum_f32x4 = wasm_f32x4_splat(0.0f);
+    nk_e4m3_t const *a_scalars = a, *b_scalars = b;
+    nk_size_t count_scalars = n;
+    nk_b32_vec_t a_raw, b_raw;
+
+nk_sqeuclidean_e4m3_v128relaxed_cycle:
     if (count_scalars < 4) {
-        nk_partial_load_b16x4_serial_(a_scalars, &a_bf16_vec, count_scalars);
-        nk_partial_load_b16x4_serial_(b_scalars, &b_bf16_vec, count_scalars);
+        a_raw = nk_partial_load_b8x4_serial_(a_scalars, count_scalars);
+        b_raw = nk_partial_load_b8x4_serial_(b_scalars, count_scalars);
         count_scalars = 0;
     }
     else {
-        nk_load_b64_serial_(a_scalars, &a_bf16_vec);
-        nk_load_b64_serial_(b_scalars, &b_bf16_vec);
+        nk_load_b32_serial_(a_scalars, &a_raw);
+        nk_load_b32_serial_(b_scalars, &b_raw);
         a_scalars += 4, b_scalars += 4, count_scalars -= 4;
     }
+    nk_b128_vec_t a_f32_vec = nk_e4m3x4_to_f32x4_v128relaxed_(a_raw);
+    nk_b128_vec_t b_f32_vec = nk_e4m3x4_to_f32x4_v128relaxed_(b_raw);
+    v128_t diff_f32x4 = wasm_f32x4_sub(a_f32_vec.v128, b_f32_vec.v128);
+    sum_f32x4 = wasm_f32x4_relaxed_madd(diff_f32x4, diff_f32x4, sum_f32x4);
+    if (count_scalars) goto nk_sqeuclidean_e4m3_v128relaxed_cycle;
 
-    // Convert bf16 → f32
-    nk_b128_vec_t a_f32_vec = nk_bf16x4_to_f32x4_v128relaxed_(a_bf16_vec);
-    nk_b128_vec_t b_f32_vec = nk_bf16x4_to_f32x4_v128relaxed_(b_bf16_vec);
+    *result = nk_reduce_add_f32x4_v128relaxed_(sum_f32x4);
+}
 
-    // Triple accumulation: ab, a², b²
+NK_PUBLIC void nk_euclidean_e4m3_v128relaxed(nk_e4m3_t const *a, nk_e4m3_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_sqeuclidean_e4m3_v128relaxed(a, b, n, result);
+    *result = nk_f32_sqrt_v128relaxed(*result);
+}
+
+NK_PUBLIC void nk_angular_e4m3_v128relaxed(nk_e4m3_t const *a, nk_e4m3_t const *b, nk_size_t n, nk_f32_t *result) {
+    v128_t ab_f32x4 = wasm_f32x4_splat(0.0f);
+    v128_t a2_f32x4 = wasm_f32x4_splat(0.0f);
+    v128_t b2_f32x4 = wasm_f32x4_splat(0.0f);
+    nk_e4m3_t const *a_scalars = a, *b_scalars = b;
+    nk_size_t count_scalars = n;
+    nk_b32_vec_t a_raw, b_raw;
+
+nk_angular_e4m3_v128relaxed_cycle:
+    if (count_scalars < 4) {
+        a_raw = nk_partial_load_b8x4_serial_(a_scalars, count_scalars);
+        b_raw = nk_partial_load_b8x4_serial_(b_scalars, count_scalars);
+        count_scalars = 0;
+    }
+    else {
+        nk_load_b32_serial_(a_scalars, &a_raw);
+        nk_load_b32_serial_(b_scalars, &b_raw);
+        a_scalars += 4, b_scalars += 4, count_scalars -= 4;
+    }
+    nk_b128_vec_t a_f32_vec = nk_e4m3x4_to_f32x4_v128relaxed_(a_raw);
+    nk_b128_vec_t b_f32_vec = nk_e4m3x4_to_f32x4_v128relaxed_(b_raw);
     ab_f32x4 = wasm_f32x4_relaxed_madd(a_f32_vec.v128, b_f32_vec.v128, ab_f32x4);
     a2_f32x4 = wasm_f32x4_relaxed_madd(a_f32_vec.v128, a_f32_vec.v128, a2_f32x4);
     b2_f32x4 = wasm_f32x4_relaxed_madd(b_f32_vec.v128, b_f32_vec.v128, b2_f32x4);
+    if (count_scalars) goto nk_angular_e4m3_v128relaxed_cycle;
 
-    if (count_scalars) goto nk_angular_bf16_v128relaxed_cycle;
+    nk_f32_t ab = nk_reduce_add_f32x4_v128relaxed_(ab_f32x4);
+    nk_f32_t a2 = nk_reduce_add_f32x4_v128relaxed_(a2_f32x4);
+    nk_f32_t b2 = nk_reduce_add_f32x4_v128relaxed_(b2_f32x4);
+    *result = (nk_f32_t)nk_angular_normalize_f64_v128relaxed_((nk_f64_t)ab, (nk_f64_t)a2, (nk_f64_t)b2);
+}
+
+NK_PUBLIC void nk_sqeuclidean_e5m2_v128relaxed(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_f32_t *result) {
+    v128_t sum_f32x4 = wasm_f32x4_splat(0.0f);
+    nk_e5m2_t const *a_scalars = a, *b_scalars = b;
+    nk_size_t count_scalars = n;
+    nk_b32_vec_t a_raw, b_raw;
+
+nk_sqeuclidean_e5m2_v128relaxed_cycle:
+    if (count_scalars < 4) {
+        a_raw = nk_partial_load_b8x4_serial_(a_scalars, count_scalars);
+        b_raw = nk_partial_load_b8x4_serial_(b_scalars, count_scalars);
+        count_scalars = 0;
+    }
+    else {
+        nk_load_b32_serial_(a_scalars, &a_raw);
+        nk_load_b32_serial_(b_scalars, &b_raw);
+        a_scalars += 4, b_scalars += 4, count_scalars -= 4;
+    }
+    nk_b128_vec_t a_f32_vec = nk_e5m2x4_to_f32x4_v128relaxed_(a_raw);
+    nk_b128_vec_t b_f32_vec = nk_e5m2x4_to_f32x4_v128relaxed_(b_raw);
+    v128_t diff_f32x4 = wasm_f32x4_sub(a_f32_vec.v128, b_f32_vec.v128);
+    sum_f32x4 = wasm_f32x4_relaxed_madd(diff_f32x4, diff_f32x4, sum_f32x4);
+    if (count_scalars) goto nk_sqeuclidean_e5m2_v128relaxed_cycle;
+
+    *result = nk_reduce_add_f32x4_v128relaxed_(sum_f32x4);
+}
+
+NK_PUBLIC void nk_euclidean_e5m2_v128relaxed(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_sqeuclidean_e5m2_v128relaxed(a, b, n, result);
+    *result = nk_f32_sqrt_v128relaxed(*result);
+}
+
+NK_PUBLIC void nk_angular_e5m2_v128relaxed(nk_e5m2_t const *a, nk_e5m2_t const *b, nk_size_t n, nk_f32_t *result) {
+    v128_t ab_f32x4 = wasm_f32x4_splat(0.0f);
+    v128_t a2_f32x4 = wasm_f32x4_splat(0.0f);
+    v128_t b2_f32x4 = wasm_f32x4_splat(0.0f);
+    nk_e5m2_t const *a_scalars = a, *b_scalars = b;
+    nk_size_t count_scalars = n;
+    nk_b32_vec_t a_raw, b_raw;
+
+nk_angular_e5m2_v128relaxed_cycle:
+    if (count_scalars < 4) {
+        a_raw = nk_partial_load_b8x4_serial_(a_scalars, count_scalars);
+        b_raw = nk_partial_load_b8x4_serial_(b_scalars, count_scalars);
+        count_scalars = 0;
+    }
+    else {
+        nk_load_b32_serial_(a_scalars, &a_raw);
+        nk_load_b32_serial_(b_scalars, &b_raw);
+        a_scalars += 4, b_scalars += 4, count_scalars -= 4;
+    }
+    nk_b128_vec_t a_f32_vec = nk_e5m2x4_to_f32x4_v128relaxed_(a_raw);
+    nk_b128_vec_t b_f32_vec = nk_e5m2x4_to_f32x4_v128relaxed_(b_raw);
+    ab_f32x4 = wasm_f32x4_relaxed_madd(a_f32_vec.v128, b_f32_vec.v128, ab_f32x4);
+    a2_f32x4 = wasm_f32x4_relaxed_madd(a_f32_vec.v128, a_f32_vec.v128, a2_f32x4);
+    b2_f32x4 = wasm_f32x4_relaxed_madd(b_f32_vec.v128, b_f32_vec.v128, b2_f32x4);
+    if (count_scalars) goto nk_angular_e5m2_v128relaxed_cycle;
 
-    // Reduce accumulators
     nk_f32_t ab = nk_reduce_add_f32x4_v128relaxed_(ab_f32x4);
     nk_f32_t a2 = nk_reduce_add_f32x4_v128relaxed_(a2_f32x4);
     nk_f32_t b2 = nk_reduce_add_f32x4_v128relaxed_(b2_f32x4);
+    *result = (nk_f32_t)nk_angular_normalize_f64_v128relaxed_((nk_f64_t)ab, (nk_f64_t)a2, (nk_f64_t)b2);
+}
 
-    // Normalize using f64 helper (handles edge cases: zero vectors, perpendicular, clamping)
+NK_PUBLIC void nk_sqeuclidean_e2m3_v128relaxed(nk_e2m3_t const *a, nk_e2m3_t const *b, nk_size_t n, nk_f32_t *result) {
+    v128_t sum_f32x4 = wasm_f32x4_splat(0.0f);
+    nk_e2m3_t const *a_scalars = a, *b_scalars = b;
+    nk_size_t count_scalars = n;
+    nk_b32_vec_t a_raw, b_raw;
+
+nk_sqeuclidean_e2m3_v128relaxed_cycle:
+    if (count_scalars < 4) {
+        a_raw = nk_partial_load_b8x4_serial_(a_scalars, count_scalars);
+        b_raw = nk_partial_load_b8x4_serial_(b_scalars, count_scalars);
+        count_scalars = 0;
+    }
+    else {
+        nk_load_b32_serial_(a_scalars, &a_raw);
+        nk_load_b32_serial_(b_scalars, &b_raw);
+        a_scalars += 4, b_scalars += 4, count_scalars -= 4;
+    }
+    nk_b128_vec_t a_f32_vec = nk_e2m3x4_to_f32x4_v128relaxed_(a_raw);
+    nk_b128_vec_t b_f32_vec = nk_e2m3x4_to_f32x4_v128relaxed_(b_raw);
+    v128_t diff_f32x4 = wasm_f32x4_sub(a_f32_vec.v128, b_f32_vec.v128);
+    sum_f32x4 = wasm_f32x4_relaxed_madd(diff_f32x4, diff_f32x4, sum_f32x4);
+    if (count_scalars) goto nk_sqeuclidean_e2m3_v128relaxed_cycle;
+
+    *result = nk_reduce_add_f32x4_v128relaxed_(sum_f32x4);
+}
+
+NK_PUBLIC void nk_euclidean_e2m3_v128relaxed(nk_e2m3_t const *a, nk_e2m3_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_sqeuclidean_e2m3_v128relaxed(a, b, n, result);
+    *result = nk_f32_sqrt_v128relaxed(*result);
+}
+
+NK_PUBLIC void nk_angular_e2m3_v128relaxed(nk_e2m3_t const *a, nk_e2m3_t const *b, nk_size_t n, nk_f32_t *result) {
+    v128_t ab_f32x4 = wasm_f32x4_splat(0.0f);
+    v128_t a2_f32x4 = wasm_f32x4_splat(0.0f);
+    v128_t b2_f32x4 = wasm_f32x4_splat(0.0f);
+    nk_e2m3_t const *a_scalars = a, *b_scalars = b;
+    nk_size_t count_scalars = n;
+    nk_b32_vec_t a_raw, b_raw;
+
+nk_angular_e2m3_v128relaxed_cycle:
+    if (count_scalars < 4) {
+        a_raw = nk_partial_load_b8x4_serial_(a_scalars, count_scalars);
+        b_raw = nk_partial_load_b8x4_serial_(b_scalars, count_scalars);
+        count_scalars = 0;
+    }
+    else {
+        nk_load_b32_serial_(a_scalars, &a_raw);
+        nk_load_b32_serial_(b_scalars, &b_raw);
+        a_scalars += 4, b_scalars += 4, count_scalars -= 4;
+    }
+    nk_b128_vec_t a_f32_vec = nk_e2m3x4_to_f32x4_v128relaxed_(a_raw);
+    nk_b128_vec_t b_f32_vec = nk_e2m3x4_to_f32x4_v128relaxed_(b_raw);
+    ab_f32x4 = wasm_f32x4_relaxed_madd(a_f32_vec.v128, b_f32_vec.v128, ab_f32x4);
+    a2_f32x4 = wasm_f32x4_relaxed_madd(a_f32_vec.v128, a_f32_vec.v128, a2_f32x4);
+    b2_f32x4 = wasm_f32x4_relaxed_madd(b_f32_vec.v128, b_f32_vec.v128, b2_f32x4);
+    if (count_scalars) goto nk_angular_e2m3_v128relaxed_cycle;
+
+    nk_f32_t ab = nk_reduce_add_f32x4_v128relaxed_(ab_f32x4);
+    nk_f32_t a2 = nk_reduce_add_f32x4_v128relaxed_(a2_f32x4);
+    nk_f32_t b2 = nk_reduce_add_f32x4_v128relaxed_(b2_f32x4);
+    *result = (nk_f32_t)nk_angular_normalize_f64_v128relaxed_((nk_f64_t)ab, (nk_f64_t)a2, (nk_f64_t)b2);
+}
+
+NK_PUBLIC void nk_sqeuclidean_e3m2_v128relaxed(nk_e3m2_t const *a, nk_e3m2_t const *b, nk_size_t n, nk_f32_t *result) {
+    v128_t sum_f32x4 = wasm_f32x4_splat(0.0f);
+    nk_e3m2_t const *a_scalars = a, *b_scalars = b;
+    nk_size_t count_scalars = n;
+    nk_b32_vec_t a_raw, b_raw;
+
+nk_sqeuclidean_e3m2_v128relaxed_cycle:
+    if (count_scalars < 4) {
+        a_raw = nk_partial_load_b8x4_serial_(a_scalars, count_scalars);
+        b_raw = nk_partial_load_b8x4_serial_(b_scalars, count_scalars);
+        count_scalars = 0;
+    }
+    else {
+        nk_load_b32_serial_(a_scalars, &a_raw);
+        nk_load_b32_serial_(b_scalars, &b_raw);
+        a_scalars += 4, b_scalars += 4, count_scalars -= 4;
+    }
+    nk_b128_vec_t a_f32_vec = nk_e3m2x4_to_f32x4_v128relaxed_(a_raw);
+    nk_b128_vec_t b_f32_vec = nk_e3m2x4_to_f32x4_v128relaxed_(b_raw);
+    v128_t diff_f32x4 = wasm_f32x4_sub(a_f32_vec.v128, b_f32_vec.v128);
+    sum_f32x4 = wasm_f32x4_relaxed_madd(diff_f32x4, diff_f32x4, sum_f32x4);
+    if (count_scalars) goto nk_sqeuclidean_e3m2_v128relaxed_cycle;
+
+    *result = nk_reduce_add_f32x4_v128relaxed_(sum_f32x4);
+}
+
+NK_PUBLIC void nk_euclidean_e3m2_v128relaxed(nk_e3m2_t const *a, nk_e3m2_t const *b, nk_size_t n, nk_f32_t *result) {
+    nk_sqeuclidean_e3m2_v128relaxed(a, b, n, result);
+    *result = nk_f32_sqrt_v128relaxed(*result);
+}
+
+NK_PUBLIC void nk_angular_e3m2_v128relaxed(nk_e3m2_t const *a, nk_e3m2_t const *b, nk_size_t n, nk_f32_t *result) {
+    v128_t ab_f32x4 = wasm_f32x4_splat(0.0f);
+    v128_t a2_f32x4 = wasm_f32x4_splat(0.0f);
+    v128_t b2_f32x4 = wasm_f32x4_splat(0.0f);
+    nk_e3m2_t const *a_scalars = a, *b_scalars = b;
+    nk_size_t count_scalars = n;
+    nk_b32_vec_t a_raw, b_raw;
+
+nk_angular_e3m2_v128relaxed_cycle:
+    if (count_scalars < 4) {
+        a_raw = nk_partial_load_b8x4_serial_(a_scalars, count_scalars);
+        b_raw = nk_partial_load_b8x4_serial_(b_scalars, count_scalars);
+        count_scalars = 0;
+    }
+    else {
+        nk_load_b32_serial_(a_scalars, &a_raw);
+        nk_load_b32_serial_(b_scalars, &b_raw);
+        a_scalars += 4, b_scalars += 4, count_scalars -= 4;
+    }
+    nk_b128_vec_t a_f32_vec = nk_e3m2x4_to_f32x4_v128relaxed_(a_raw);
+    nk_b128_vec_t b_f32_vec = nk_e3m2x4_to_f32x4_v128relaxed_(b_raw);
+    ab_f32x4 = wasm_f32x4_relaxed_madd(a_f32_vec.v128, b_f32_vec.v128, ab_f32x4);
+    a2_f32x4 = wasm_f32x4_relaxed_madd(a_f32_vec.v128, a_f32_vec.v128, a2_f32x4);
+    b2_f32x4 = wasm_f32x4_relaxed_madd(b_f32_vec.v128, b_f32_vec.v128, b2_f32x4);
+    if (count_scalars) goto nk_angular_e3m2_v128relaxed_cycle;
+
+    nk_f32_t ab = nk_reduce_add_f32x4_v128relaxed_(ab_f32x4);
+    nk_f32_t a2 = nk_reduce_add_f32x4_v128relaxed_(a2_f32x4);
+    nk_f32_t b2 = nk_reduce_add_f32x4_v128relaxed_(b2_f32x4);
     *result = (nk_f32_t)nk_angular_normalize_f64_v128relaxed_((nk_f64_t)ab, (nk_f64_t)a2, (nk_f64_t)b2);
 }
 
-#pragma endregion - Smaller Floats
-#pragma region - Spatial From-Dot Helpers
+#pragma endregion FP8 Floats
+#pragma region Spatial From Dot Helpers
 
 /** @brief Angular from_dot: computes 1 − dot / √(query_sumsq × target_sumsq) for 4 pairs in f32. */
 NK_INTERNAL void nk_angular_through_f32_from_dot_v128relaxed_(nk_b128_vec_t dots, nk_f32_t query_sumsq,
@@ -437,8 +689,8 @@ NK_INTERNAL void nk_euclidean_through_u32_from_dot_v128relaxed_(nk_b128_vec_t do
     results->v128 = wasm_f32x4_sqrt(dist_sq_f32x4);
 }
 
-#pragma endregion - Spatial From - Dot Helpers
-#pragma region - Integer Spatial
+#pragma endregion Spatial From Dot Helpers
+#pragma region I8 and U8 Integers
 
 NK_PUBLIC void nk_sqeuclidean_u8_v128relaxed(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_u32_t *result) {
     v128_t sum_u32x4 = wasm_u32x4_splat(0);
@@ -703,7 +955,7 @@ NK_PUBLIC void nk_angular_i8_v128relaxed(nk_i8_t const *a, nk_i8_t const *b, nk_
     *result = (nk_f32_t)nk_angular_normalize_f64_v128relaxed_(dot_ab, norm_aa, norm_bb);
 }
 
-#pragma endregion - Integer Spatial
+#pragma endregion I8 and U8 Integers
 
 #if defined(__clang__)
 #pragma clang attribute pop