numkong 7.0.0

package/include/numkong/spatial/rvvbf16.h

@@ -0,0 +1,123 @@
+/**
+ *  @brief SIMD-accelerated Spatial Similarity Measures for RISC-V BF16.
+ *  @file include/numkong/spatial/rvvbf16.h
+ *  @author Ash Vardanian
+ *  @date January 5, 2026
+ *
+ *  @sa include/numkong/spatial.h
+ *
+ *  Zvfbfwma provides widening bf16 fused multiply-accumulate to f32:
+ *    vfwmaccbf16: f32 ← bf16 × bf16
+ *
+ *  For L2 distance, we use the identity: (a−b)² = a² + b² − 2 × a × b
+ *  This allows us to use vfwmaccbf16 for all computations.
+ *
+ *  Requires: RVV 1.0 + Zvfbfwma extension (GCC 14+ or Clang 18+)
+ */
+#ifndef NK_SPATIAL_RVVBF16_H
+#define NK_SPATIAL_RVVBF16_H
+
+#if NK_TARGET_RISCV_
+#if NK_TARGET_RVVBF16
+
+#include "numkong/types.h"
+#include "numkong/spatial/rvv.h" // `nk_f32_sqrt_rvv`
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=+v,+zvfbfwma"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=+v,+zvfbfwma")
+#endif
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+NK_PUBLIC void nk_sqeuclidean_bf16_rvvbf16(nk_bf16_t const *a_scalars, nk_bf16_t const *b_scalars,
+                                           nk_size_t count_scalars, nk_f32_t *result) {
+    // Per-lane accumulators — deferred horizontal reduction
+    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
+    vfloat32m2_t sq_sum_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax); // a² + b²
+    vfloat32m2_t ab_sum_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax); // a × b
+
+    for (nk_size_t vector_length; count_scalars > 0;
+         count_scalars -= vector_length, a_scalars += vector_length, b_scalars += vector_length) {
+        vector_length = __riscv_vsetvl_e16m1(count_scalars);
+        vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1((unsigned short const *)a_scalars, vector_length);
+        vuint16m1_t b_u16m1 = __riscv_vle16_v_u16m1((unsigned short const *)b_scalars, vector_length);
+        vbfloat16m1_t a_bf16m1 = __riscv_vreinterpret_v_u16m1_bf16m1(a_u16m1);
+        vbfloat16m1_t b_bf16m1 = __riscv_vreinterpret_v_u16m1_bf16m1(b_u16m1);
+
+        // Accumulate a², b², and a×b per-lane (no per-iteration reduction)
+        sq_sum_f32m2 = __riscv_vfwmaccbf16_vv_f32m2_tu(sq_sum_f32m2, a_bf16m1, a_bf16m1, vector_length);
+        sq_sum_f32m2 = __riscv_vfwmaccbf16_vv_f32m2_tu(sq_sum_f32m2, b_bf16m1, b_bf16m1, vector_length);
+        ab_sum_f32m2 = __riscv_vfwmaccbf16_vv_f32m2_tu(ab_sum_f32m2, a_bf16m1, b_bf16m1, vector_length);
+    }
+
+    // Single horizontal reduction after the loop
+    vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
+    nk_f32_t sq_sum = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m2_f32m1(sq_sum_f32m2, zero_f32m1, vlmax));
+    nk_f32_t ab_sum = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m2_f32m1(ab_sum_f32m2, zero_f32m1, vlmax));
+    *result = sq_sum - 2.0f * ab_sum;
+}
+
+NK_PUBLIC void nk_euclidean_bf16_rvvbf16(nk_bf16_t const *a_scalars, nk_bf16_t const *b_scalars,
+                                         nk_size_t count_scalars, nk_f32_t *result) {
+    nk_sqeuclidean_bf16_rvvbf16(a_scalars, b_scalars, count_scalars, result);
+    // Handle potential negative values from floating point errors
+    *result = *result > 0.0f ? nk_f32_sqrt_rvv(*result) : 0.0f;
+}
+
+NK_PUBLIC void nk_angular_bf16_rvvbf16(nk_bf16_t const *a_scalars, nk_bf16_t const *b_scalars, nk_size_t count_scalars,
+                                       nk_f32_t *result) {
+    // Per-lane accumulators — deferred horizontal reduction
+    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
+    vfloat32m2_t dot_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
+    vfloat32m2_t a_sq_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
+    vfloat32m2_t b_sq_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
+
+    for (nk_size_t vector_length; count_scalars > 0;
+         count_scalars -= vector_length, a_scalars += vector_length, b_scalars += vector_length) {
+        vector_length = __riscv_vsetvl_e16m1(count_scalars);
+        vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1((unsigned short const *)a_scalars, vector_length);
+        vuint16m1_t b_u16m1 = __riscv_vle16_v_u16m1((unsigned short const *)b_scalars, vector_length);
+        vbfloat16m1_t a_bf16m1 = __riscv_vreinterpret_v_u16m1_bf16m1(a_u16m1);
+        vbfloat16m1_t b_bf16m1 = __riscv_vreinterpret_v_u16m1_bf16m1(b_u16m1);
+
+        // dot += a × b
+        dot_f32m2 = __riscv_vfwmaccbf16_vv_f32m2_tu(dot_f32m2, a_bf16m1, b_bf16m1, vector_length);
+        // a_sq += a × a
+        a_sq_f32m2 = __riscv_vfwmaccbf16_vv_f32m2_tu(a_sq_f32m2, a_bf16m1, a_bf16m1, vector_length);
+        // b_sq += b × b
+        b_sq_f32m2 = __riscv_vfwmaccbf16_vv_f32m2_tu(b_sq_f32m2, b_bf16m1, b_bf16m1, vector_length);
+    }
+
+    // Single horizontal reduction after the loop
+    vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
+    nk_f32_t dot = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m2_f32m1(dot_f32m2, zero_f32m1, vlmax));
+    nk_f32_t a_sq = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m2_f32m1(a_sq_f32m2, zero_f32m1, vlmax));
+    nk_f32_t b_sq = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m2_f32m1(b_sq_f32m2, zero_f32m1, vlmax));
+
+    // Normalize: 1 − dot / sqrt(‖a‖² × ‖b‖²)
+    if (a_sq == 0.0f && b_sq == 0.0f) { *result = 0.0f; }
+    else if (dot == 0.0f) { *result = 1.0f; }
+    else {
+        nk_f32_t unclipped = 1.0f - dot * nk_f32_rsqrt_rvv(a_sq) * nk_f32_rsqrt_rvv(b_sq);
+        *result = unclipped > 0.0f ? unclipped : 0.0f;
+    }
+}
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#endif // NK_TARGET_RVVBF16
+#endif // NK_TARGET_RISCV_
+#endif // NK_SPATIAL_RVVBF16_H

package/include/numkong/spatial/rvvhalf.h

@@ -0,0 +1,117 @@
+/**
+ *  @brief SIMD-accelerated Spatial Similarity Measures for RISC-V FP16.
+ *  @file include/numkong/spatial/rvvhalf.h
+ *  @author Ash Vardanian
+ *  @date January 5, 2026
+ *
+ *  @sa include/numkong/spatial.h
+ *
+ *  Zvfh provides native half-precision (f16) vector operations.
+ *  Uses widening operations (f16 → f32) for precision accumulation.
+ *
+ *  Requires: RVV 1.0 + Zvfh extension (GCC 14+ or Clang 18+)
+ */
+#ifndef NK_SPATIAL_RVVHALF_H
+#define NK_SPATIAL_RVVHALF_H
+
+#if NK_TARGET_RISCV_
+#if NK_TARGET_RVVHALF
+
+#include "numkong/types.h"
+#include "numkong/spatial/rvv.h" // `nk_f32_sqrt_rvv`
+
+#if defined(__clang__)
+#pragma clang attribute push(__attribute__((target("arch=+v,+zvfh"))), apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("arch=+v,+zvfh")
+#endif
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+NK_PUBLIC void nk_sqeuclidean_f16_rvvhalf(nk_f16_t const *a_scalars, nk_f16_t const *b_scalars, nk_size_t count_scalars,
+                                          nk_f32_t *result) {
+    // Per-lane accumulator — deferred horizontal reduction
+    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
+    vfloat32m2_t sum_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
+
+    for (nk_size_t vector_length; count_scalars > 0;
+         count_scalars -= vector_length, a_scalars += vector_length, b_scalars += vector_length) {
+        vector_length = __riscv_vsetvl_e16m1(count_scalars);
+        vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1((unsigned short const *)a_scalars, vector_length);
+        vuint16m1_t b_u16m1 = __riscv_vle16_v_u16m1((unsigned short const *)b_scalars, vector_length);
+        vfloat16m1_t a_f16m1 = __riscv_vreinterpret_v_u16m1_f16m1(a_u16m1);
+        vfloat16m1_t b_f16m1 = __riscv_vreinterpret_v_u16m1_f16m1(b_u16m1);
+        // Upcast to f32 before subtraction to avoid catastrophic cancellation in f16
+        vfloat32m2_t a_f32m2 = __riscv_vfwcvt_f_f_v_f32m2(a_f16m1, vector_length);
+        vfloat32m2_t b_f32m2 = __riscv_vfwcvt_f_f_v_f32m2(b_f16m1, vector_length);
+        vfloat32m2_t diff_f32m2 = __riscv_vfsub_vv_f32m2(a_f32m2, b_f32m2, vector_length);
+        // Accumulate diff² in f32
+        sum_f32m2 = __riscv_vfmacc_vv_f32m2_tu(sum_f32m2, diff_f32m2, diff_f32m2, vector_length);
+    }
+
+    // Single horizontal reduction after the loop
+    vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
+    *result = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m2_f32m1(sum_f32m2, zero_f32m1, vlmax));
+}
+
+NK_PUBLIC void nk_euclidean_f16_rvvhalf(nk_f16_t const *a_scalars, nk_f16_t const *b_scalars, nk_size_t count_scalars,
+                                        nk_f32_t *result) {
+    nk_sqeuclidean_f16_rvvhalf(a_scalars, b_scalars, count_scalars, result);
+    *result = nk_f32_sqrt_rvv(*result);
+}
+
+NK_PUBLIC void nk_angular_f16_rvvhalf(nk_f16_t const *a_scalars, nk_f16_t const *b_scalars, nk_size_t count_scalars,
+                                      nk_f32_t *result) {
+    // Per-lane accumulators — deferred horizontal reduction
+    nk_size_t vlmax = __riscv_vsetvlmax_e32m2();
+    vfloat32m2_t dot_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
+    vfloat32m2_t a_sq_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
+    vfloat32m2_t b_sq_f32m2 = __riscv_vfmv_v_f_f32m2(0.0f, vlmax);
+
+    for (nk_size_t vector_length; count_scalars > 0;
+         count_scalars -= vector_length, a_scalars += vector_length, b_scalars += vector_length) {
+        vector_length = __riscv_vsetvl_e16m1(count_scalars);
+        vuint16m1_t a_u16m1 = __riscv_vle16_v_u16m1((unsigned short const *)a_scalars, vector_length);
+        vuint16m1_t b_u16m1 = __riscv_vle16_v_u16m1((unsigned short const *)b_scalars, vector_length);
+        vfloat16m1_t a_f16m1 = __riscv_vreinterpret_v_u16m1_f16m1(a_u16m1);
+        vfloat16m1_t b_f16m1 = __riscv_vreinterpret_v_u16m1_f16m1(b_u16m1);
+
+        // dot += a × b (widened to f32)
+        dot_f32m2 = __riscv_vfwmacc_vv_f32m2_tu(dot_f32m2, a_f16m1, b_f16m1, vector_length);
+        // a_sq += a × a
+        a_sq_f32m2 = __riscv_vfwmacc_vv_f32m2_tu(a_sq_f32m2, a_f16m1, a_f16m1, vector_length);
+        // b_sq += b × b
+        b_sq_f32m2 = __riscv_vfwmacc_vv_f32m2_tu(b_sq_f32m2, b_f16m1, b_f16m1, vector_length);
+    }
+
+    // Single horizontal reduction after the loop
+    vfloat32m1_t zero_f32m1 = __riscv_vfmv_v_f_f32m1(0.0f, 1);
+    nk_f32_t dot = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m2_f32m1(dot_f32m2, zero_f32m1, vlmax));
+    nk_f32_t a_sq = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m2_f32m1(a_sq_f32m2, zero_f32m1, vlmax));
+    nk_f32_t b_sq = __riscv_vfmv_f_s_f32m1_f32(__riscv_vfredusum_vs_f32m2_f32m1(b_sq_f32m2, zero_f32m1, vlmax));
+
+    // Normalize: 1 − dot / sqrt(‖a‖² × ‖b‖²)
+    if (a_sq == 0.0f && b_sq == 0.0f) { *result = 0.0f; }
+    else if (dot == 0.0f) { *result = 1.0f; }
+    else {
+        nk_f32_t unclipped = 1.0f - dot * nk_f32_rsqrt_rvv(a_sq) * nk_f32_rsqrt_rvv(b_sq);
+        *result = unclipped > 0.0f ? unclipped : 0.0f;
+    }
+}
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#endif // NK_TARGET_RVVHALF
+#endif // NK_TARGET_RISCV_
+#endif // NK_SPATIAL_RVVHALF_H

package/include/numkong/spatial/sapphire.h

@@ -0,0 +1,343 @@
+/**
+ *  @brief SIMD-accelerated Spatial Similarity Measures for Sapphire Rapids.
+ *  @file include/numkong/spatial/sapphire.h
+ *  @author Ash Vardanian
+ *  @date December 27, 2025
+ *
+ *  @sa include/numkong/spatial.h
+ *
+ *  Sapphire Rapids adds native FP16 support via AVX-512 FP16 extension.
+ *  For e4m3 L2 distance, we can leverage F16 for the subtraction step:
+ *  - e4m3 differences fit in F16 (max |a−b| = 896 < 65504)
+ *  - But squared differences overflow F16 (896² = 802816 > 65504)
+ *  - So: subtract in F16, convert to F32, then square and accumulate
+ *
+ *  For e2m3/e3m2 L2 distance, squared differences fit in FP16:
+ *  - E2M3: max |a−b| = 15, max (a−b)² = 225 < 65504, flush cadence = 4 (conservative for uniformity)
+ *  - E3M2: max |a−b| = 56, max (a−b)² = 3136 < 65504, flush cadence = 4
+ *  So the entire sub+square+accumulate stays in FP16 with periodic F32 flush.
+ *
+ *  @section spatial_sapphire_instructions Relevant Instructions
+ *
+ *      Intrinsic                   Instruction                     Sapphire    Genoa
+ *      _mm256_sub_ph               VSUBPH (YMM, YMM, YMM)          4cy @ p05   3cy @ p01
+ *      _mm512_cvtph_ps             VCVTPH2PS (ZMM, YMM)            5cy @ p05   5cy @ p01
+ *      _mm512_fmadd_ps             VFMADD (ZMM, ZMM, ZMM)          4cy @ p05   4cy @ p01
+ *      _mm512_reduce_add_ps        (pseudo: VHADDPS chain)         ~8cy        ~8cy
+ *      _mm_maskz_loadu_epi8        VMOVDQU8 (XMM {K}, M128)        7cy @ p23   7cy @ p23
+ */
+#ifndef NK_SPATIAL_SAPPHIRE_H
+#define NK_SPATIAL_SAPPHIRE_H
+
+#if NK_TARGET_X86_
+#if NK_TARGET_SAPPHIRE
+
+#include "numkong/types.h"
+#include "numkong/cast/sapphire.h"   // `nk_e4m3x16_to_f16x16_sapphire_`
+#include "numkong/dot/sapphire.h"    // `nk_e2m3x32_to_f16x32_sapphire_`, `nk_flush_f16_to_f32_sapphire_`
+#include "numkong/spatial/haswell.h" // `nk_angular_normalize_f32_haswell_`, `nk_f32_sqrt_haswell`
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+#if defined(__clang__)
+#pragma clang attribute push(                                                                        \
+    __attribute__((target("avx2,avx512f,avx512vl,avx512bw,avx512dq,avx512fp16,f16c,fma,bmi,bmi2"))), \
+    apply_to = function)
+#elif defined(__GNUC__)
+#pragma GCC push_options
+#pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "avx512dq", "avx512fp16", "f16c", "fma", "bmi", "bmi2")
+#endif
+
+NK_PUBLIC void nk_sqeuclidean_e4m3_sapphire(nk_e4m3_t const *a_scalars, nk_e4m3_t const *b_scalars,
+                                            nk_size_t count_scalars, nk_f32_t *result) {
+    __m512 sum_f32x16 = _mm512_setzero_ps();
+
+    while (count_scalars > 0) {
+        nk_size_t const n = count_scalars < 16 ? count_scalars : 16;
+        __mmask16 const mask = (__mmask16)_bzhi_u32(0xFFFF, n);
+        __m128i a_e4m3x16 = _mm_maskz_loadu_epi8(mask, a_scalars);
+        __m128i b_e4m3x16 = _mm_maskz_loadu_epi8(mask, b_scalars);
+
+        // Convert e4m3 → f16
+        __m256h a_f16x16 = nk_e4m3x16_to_f16x16_sapphire_(a_e4m3x16);
+        __m256h b_f16x16 = nk_e4m3x16_to_f16x16_sapphire_(b_e4m3x16);
+
+        // Subtract in F16 − differences fit (max 896 < 65504)
+        __m256h diff_f16x16 = _mm256_sub_ph(a_f16x16, b_f16x16);
+
+        // Convert to F32 before squaring (896² = 802816 overflows F16!)
+        __m512 diff_f32x16 = _mm512_cvtph_ps(_mm256_castph_si256(diff_f16x16));
+
+        // Square and accumulate in F32
+        sum_f32x16 = _mm512_fmadd_ps(diff_f32x16, diff_f32x16, sum_f32x16);
+        a_scalars += n, b_scalars += n, count_scalars -= n;
+    }
+
+    *result = _mm512_reduce_add_ps(sum_f32x16);
+}
+
+NK_PUBLIC void nk_euclidean_e4m3_sapphire(nk_e4m3_t const *a_scalars, nk_e4m3_t const *b_scalars,
+                                          nk_size_t count_scalars, nk_f32_t *result) {
+    nk_sqeuclidean_e4m3_sapphire(a_scalars, b_scalars, count_scalars, result);
+    *result = nk_f32_sqrt_haswell(*result);
+}
+
+NK_PUBLIC void nk_sqeuclidean_e2m3_sapphire(nk_e2m3_t const *a_scalars, nk_e2m3_t const *b_scalars,
+                                            nk_size_t count_scalars, nk_f32_t *result) {
+    __m512 sum_f32x16 = _mm512_setzero_ps();
+
+    // Main loop: 4-way unrolled, 128 elements per flush
+    while (count_scalars >= 128) {
+        __m512h acc_f16x32 = _mm512_setzero_ph();
+        __m512h a_f16x32, b_f16x32, diff_f16x32;
+        // Iteration 1
+        a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars));
+        b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars));
+        diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
+        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
+        // Iteration 2
+        a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 32));
+        b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 32));
+        diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
+        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
+        // Iteration 3
+        a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 64));
+        b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 64));
+        diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
+        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
+        // Iteration 4
+        a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 96));
+        b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 96));
+        diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
+        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
+        // Flush to F32
+        sum_f32x16 = nk_flush_f16_to_f32_sapphire_(acc_f16x32, sum_f32x16);
+        a_scalars += 128, b_scalars += 128, count_scalars -= 128;
+    }
+
+    // Tail: remaining 0–127 elements, 32 at a time via masked loads
+    __m512h acc_f16x32 = _mm512_setzero_ph();
+    while (count_scalars > 0) {
+        nk_size_t const n = count_scalars < 32 ? count_scalars : 32;
+        __mmask32 const mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
+        __m512h a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_maskz_loadu_epi8(mask, a_scalars));
+        __m512h b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_maskz_loadu_epi8(mask, b_scalars));
+        __m512h diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
+        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
+        a_scalars += n, b_scalars += n, count_scalars -= n;
+    }
+    sum_f32x16 = nk_flush_f16_to_f32_sapphire_(acc_f16x32, sum_f32x16);
+
+    *result = nk_reduce_add_f32x16_skylake_(sum_f32x16);
+}
+
+NK_PUBLIC void nk_sqeuclidean_e3m2_sapphire(nk_e3m2_t const *a_scalars, nk_e3m2_t const *b_scalars,
+                                            nk_size_t count_scalars, nk_f32_t *result) {
+    __m512 sum_f32x16 = _mm512_setzero_ps();
+
+    // Main loop: 4-way unrolled, 128 elements per flush
+    while (count_scalars >= 128) {
+        __m512h acc_f16x32 = _mm512_setzero_ph();
+        __m512h a_f16x32, b_f16x32, diff_f16x32;
+        // Iteration 1
+        a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars));
+        b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars));
+        diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
+        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
+        // Iteration 2
+        a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 32));
+        b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 32));
+        diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
+        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
+        // Iteration 3
+        a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 64));
+        b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 64));
+        diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
+        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
+        // Iteration 4
+        a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 96));
+        b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 96));
+        diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
+        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
+        // Flush to F32
+        sum_f32x16 = nk_flush_f16_to_f32_sapphire_(acc_f16x32, sum_f32x16);
+        a_scalars += 128, b_scalars += 128, count_scalars -= 128;
+    }
+
+    // Tail: remaining 0–127 elements, 32 at a time via masked loads
+    __m512h acc_f16x32 = _mm512_setzero_ph();
+    while (count_scalars > 0) {
+        nk_size_t const n = count_scalars < 32 ? count_scalars : 32;
+        __mmask32 const mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
+        __m512h a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_maskz_loadu_epi8(mask, a_scalars));
+        __m512h b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_maskz_loadu_epi8(mask, b_scalars));
+        __m512h diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
+        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
+        a_scalars += n, b_scalars += n, count_scalars -= n;
+    }
+    sum_f32x16 = nk_flush_f16_to_f32_sapphire_(acc_f16x32, sum_f32x16);
+
+    *result = nk_reduce_add_f32x16_skylake_(sum_f32x16);
+}
+
+NK_PUBLIC void nk_euclidean_e2m3_sapphire(nk_e2m3_t const *a_scalars, nk_e2m3_t const *b_scalars,
+                                          nk_size_t count_scalars, nk_f32_t *result) {
+    nk_sqeuclidean_e2m3_sapphire(a_scalars, b_scalars, count_scalars, result);
+    *result = nk_f32_sqrt_haswell(*result);
+}
+
+NK_PUBLIC void nk_euclidean_e3m2_sapphire(nk_e3m2_t const *a_scalars, nk_e3m2_t const *b_scalars,
+                                          nk_size_t count_scalars, nk_f32_t *result) {
+    nk_sqeuclidean_e3m2_sapphire(a_scalars, b_scalars, count_scalars, result);
+    *result = nk_f32_sqrt_haswell(*result);
+}
+
+NK_PUBLIC void nk_angular_e2m3_sapphire(nk_e2m3_t const *a_scalars, nk_e2m3_t const *b_scalars, nk_size_t count_scalars,
+                                        nk_f32_t *result) {
+    __m512 sum_dot_f32x16 = _mm512_setzero_ps();
+    __m512 sum_a_f32x16 = _mm512_setzero_ps();
+    __m512 sum_b_f32x16 = _mm512_setzero_ps();
+
+    // Main loop: 4-way unrolled, 128 elements per flush
+    while (count_scalars >= 128) {
+        __m512h dot_acc = _mm512_setzero_ph();
+        __m512h a_norm_acc = _mm512_setzero_ph();
+        __m512h b_norm_acc = _mm512_setzero_ph();
+        __m512h a_f16x32, b_f16x32;
+        // Iteration 1
+        a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars));
+        b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars));
+        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
+        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
+        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
+        // Iteration 2
+        a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 32));
+        b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 32));
+        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
+        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
+        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
+        // Iteration 3
+        a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 64));
+        b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 64));
+        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
+        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
+        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
+        // Iteration 4
+        a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 96));
+        b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 96));
+        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
+        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
+        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
+        // Flush to F32
+        sum_dot_f32x16 = nk_flush_f16_to_f32_sapphire_(dot_acc, sum_dot_f32x16);
+        sum_a_f32x16 = nk_flush_f16_to_f32_sapphire_(a_norm_acc, sum_a_f32x16);
+        sum_b_f32x16 = nk_flush_f16_to_f32_sapphire_(b_norm_acc, sum_b_f32x16);
+        a_scalars += 128, b_scalars += 128, count_scalars -= 128;
+    }
+
+    // Tail: remaining 0–127 elements, 32 at a time via masked loads
+    __m512h dot_acc = _mm512_setzero_ph();
+    __m512h a_norm_acc = _mm512_setzero_ph();
+    __m512h b_norm_acc = _mm512_setzero_ph();
+    while (count_scalars > 0) {
+        nk_size_t const n = count_scalars < 32 ? count_scalars : 32;
+        __mmask32 const mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
+        __m512h a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_maskz_loadu_epi8(mask, a_scalars));
+        __m512h b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_maskz_loadu_epi8(mask, b_scalars));
+        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
+        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
+        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
+        a_scalars += n, b_scalars += n, count_scalars -= n;
+    }
+    sum_dot_f32x16 = nk_flush_f16_to_f32_sapphire_(dot_acc, sum_dot_f32x16);
+    sum_a_f32x16 = nk_flush_f16_to_f32_sapphire_(a_norm_acc, sum_a_f32x16);
+    sum_b_f32x16 = nk_flush_f16_to_f32_sapphire_(b_norm_acc, sum_b_f32x16);
+
+    nk_f32_t dot_f32 = nk_reduce_add_f32x16_skylake_(sum_dot_f32x16);
+    nk_f32_t a_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(sum_a_f32x16);
+    nk_f32_t b_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(sum_b_f32x16);
+    *result = nk_angular_normalize_f32_haswell_(dot_f32, a_norm_sq_f32, b_norm_sq_f32);
+}
+
+NK_PUBLIC void nk_angular_e3m2_sapphire(nk_e3m2_t const *a_scalars, nk_e3m2_t const *b_scalars, nk_size_t count_scalars,
+                                        nk_f32_t *result) {
+    __m512 sum_dot_f32x16 = _mm512_setzero_ps();
+    __m512 sum_a_f32x16 = _mm512_setzero_ps();
+    __m512 sum_b_f32x16 = _mm512_setzero_ps();
+
+    // Main loop: 4-way unrolled, 128 elements per flush
+    while (count_scalars >= 128) {
+        __m512h dot_acc = _mm512_setzero_ph();
+        __m512h a_norm_acc = _mm512_setzero_ph();
+        __m512h b_norm_acc = _mm512_setzero_ph();
+        __m512h a_f16x32, b_f16x32;
+        // Iteration 1
+        a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars));
+        b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars));
+        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
+        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
+        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
+        // Iteration 2
+        a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 32));
+        b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 32));
+        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
+        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
+        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
+        // Iteration 3
+        a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 64));
+        b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 64));
+        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
+        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
+        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
+        // Iteration 4
+        a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 96));
+        b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 96));
+        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
+        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
+        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
+        // Flush to F32
+        sum_dot_f32x16 = nk_flush_f16_to_f32_sapphire_(dot_acc, sum_dot_f32x16);
+        sum_a_f32x16 = nk_flush_f16_to_f32_sapphire_(a_norm_acc, sum_a_f32x16);
+        sum_b_f32x16 = nk_flush_f16_to_f32_sapphire_(b_norm_acc, sum_b_f32x16);
+        a_scalars += 128, b_scalars += 128, count_scalars -= 128;
+    }
+
+    // Tail: remaining 0–127 elements, 32 at a time via masked loads
+    __m512h dot_acc = _mm512_setzero_ph();
+    __m512h a_norm_acc = _mm512_setzero_ph();
+    __m512h b_norm_acc = _mm512_setzero_ph();
+    while (count_scalars > 0) {
+        nk_size_t const n = count_scalars < 32 ? count_scalars : 32;
+        __mmask32 const mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
+        __m512h a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_maskz_loadu_epi8(mask, a_scalars));
+        __m512h b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_maskz_loadu_epi8(mask, b_scalars));
+        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
+        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
+        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
+        a_scalars += n, b_scalars += n, count_scalars -= n;
+    }
+    sum_dot_f32x16 = nk_flush_f16_to_f32_sapphire_(dot_acc, sum_dot_f32x16);
+    sum_a_f32x16 = nk_flush_f16_to_f32_sapphire_(a_norm_acc, sum_a_f32x16);
+    sum_b_f32x16 = nk_flush_f16_to_f32_sapphire_(b_norm_acc, sum_b_f32x16);
+
+    nk_f32_t dot_f32 = nk_reduce_add_f32x16_skylake_(sum_dot_f32x16);
+    nk_f32_t a_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(sum_a_f32x16);
+    nk_f32_t b_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(sum_b_f32x16);
+    *result = nk_angular_normalize_f32_haswell_(dot_f32, a_norm_sq_f32, b_norm_sq_f32);
+}
+
+#if defined(__clang__)
+#pragma clang attribute pop
+#elif defined(__GNUC__)
+#pragma GCC pop_options
+#endif
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // NK_TARGET_SAPPHIRE
+#endif // NK_TARGET_X86_
+#endif // NK_SPATIAL_SAPPHIRE_H