numkong 7.0.0 → 7.4.2

package/include/numkong/reduce/neonhalf.h

@@ -1,157 +0,0 @@
-/**
- *  @brief NEON FP16 implementations for the redesigned reduction API (moments + minmax).
- *  @file include/numkong/reduce/neonhalf.h
- *  @author Ash Vardanian
- *  @date February 13, 2026
- *
- *  @sa include/numkong/reduce.h
- *
- *  @section reduce_neonhalf_new_design Design Notes
- *
- *  Moments (sum + sum-of-squares) accumulate in f32 via vcvt_f32_f16 widening, giving
- *  full f32 precision. The contiguous path processes 8 f16 elements per iteration, widening
- *  to two f32x4 halves and using vfmaq_f32 for fused multiply-accumulate of squares.
- *
- *  Minmax tracks min/max values as native f16x8 with u16x8 iteration counters (same width
- *  as f16). The u16 counters wrap at 65536, so the dispatcher splits arrays larger than
- *  65536 * 8 = 524288 elements via recursive halving.
- */
-#ifndef NK_REDUCE_NEONHALF_H
-#define NK_REDUCE_NEONHALF_H
-
-#if NK_TARGET_ARM_
-#if NK_TARGET_NEONHALF
-
-#include "numkong/types.h"
-#include "numkong/cast/neon.h"
-#include "numkong/cast/serial.h"
-#include "numkong/reduce/serial.h"
-
-#if defined(__cplusplus)
-extern "C" {
-#endif
-
-#if defined(__clang__)
-#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
-#elif defined(__GNUC__)
-#pragma GCC push_options
-#pragma GCC target("arch=armv8.2-a+simd+fp16")
-#endif
-
-NK_INTERNAL void nk_reduce_moments_f16_neonhalf_contiguous_( //
-    nk_f16_t const *data_ptr, nk_size_t count,               //
-    nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
-    float32x4_t sum_f32x4 = vdupq_n_f32(0);
-    float32x4_t sumsq_f32x4 = vdupq_n_f32(0);
-    nk_size_t idx = 0;
-
-    for (; idx + 8 <= count; idx += 8) {
-        float16x8_t data_f16x8 = vld1q_f16((nk_f16_for_arm_simd_t const *)(data_ptr + idx));
-        float32x4_t low_f32x4 = vcvt_f32_f16(vget_low_f16(data_f16x8));
-        float32x4_t high_f32x4 = vcvt_f32_f16(vget_high_f16(data_f16x8));
-        sum_f32x4 = vaddq_f32(sum_f32x4, low_f32x4);
-        sum_f32x4 = vaddq_f32(sum_f32x4, high_f32x4);
-        sumsq_f32x4 = vfmaq_f32(sumsq_f32x4, low_f32x4, low_f32x4);
-        sumsq_f32x4 = vfmaq_f32(sumsq_f32x4, high_f32x4, high_f32x4);
-    }
-
-    // Scalar tail
-    nk_f32_t sum = vaddvq_f32(sum_f32x4);
-    nk_f32_t sumsq = vaddvq_f32(sumsq_f32x4);
-    for (; idx < count; ++idx) {
-        nk_f32_t value_f32;
-        nk_f16_to_f32_serial(data_ptr + idx, &value_f32);
-        sum += value_f32, sumsq += value_f32 * value_f32;
-    }
-    *sum_ptr = sum, *sumsq_ptr = sumsq;
-}
-
-NK_INTERNAL void nk_reduce_moments_f16_neonhalf_strided_(                 //
-    nk_f16_t const *data_ptr, nk_size_t count, nk_size_t stride_elements, //
-    nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
-    float32x4_t sum_f32x4 = vdupq_n_f32(0);
-    float32x4_t sumsq_f32x4 = vdupq_n_f32(0);
-    nk_size_t idx = 0;
-
-    if (stride_elements == 2) {
-        for (; idx + 8 <= count; idx += 8) {
-            uint16x8x2_t loaded_u16x8x2 = vld2q_u16((uint16_t const *)(data_ptr + idx * 2));
-            float16x8_t data_f16x8 = vreinterpretq_f16_u16(loaded_u16x8x2.val[0]);
-            float32x4_t low_f32x4 = vcvt_f32_f16(vget_low_f16(data_f16x8));
-            float32x4_t high_f32x4 = vcvt_f32_f16(vget_high_f16(data_f16x8));
-            sum_f32x4 = vaddq_f32(sum_f32x4, low_f32x4);
-            sum_f32x4 = vaddq_f32(sum_f32x4, high_f32x4);
-            sumsq_f32x4 = vfmaq_f32(sumsq_f32x4, low_f32x4, low_f32x4);
-            sumsq_f32x4 = vfmaq_f32(sumsq_f32x4, high_f32x4, high_f32x4);
-        }
-    }
-    else if (stride_elements == 3) {
-        for (; idx + 8 <= count; idx += 8) {
-            uint16x8x3_t loaded_u16x8x3 = vld3q_u16((uint16_t const *)(data_ptr + idx * 3));
-            float16x8_t data_f16x8 = vreinterpretq_f16_u16(loaded_u16x8x3.val[0]);
-            float32x4_t low_f32x4 = vcvt_f32_f16(vget_low_f16(data_f16x8));
-            float32x4_t high_f32x4 = vcvt_f32_f16(vget_high_f16(data_f16x8));
-            sum_f32x4 = vaddq_f32(sum_f32x4, low_f32x4);
-            sum_f32x4 = vaddq_f32(sum_f32x4, high_f32x4);
-            sumsq_f32x4 = vfmaq_f32(sumsq_f32x4, low_f32x4, low_f32x4);
-            sumsq_f32x4 = vfmaq_f32(sumsq_f32x4, high_f32x4, high_f32x4);
-        }
-    }
-    else if (stride_elements == 4) {
-        for (; idx + 8 <= count; idx += 8) {
-            uint16x8x4_t loaded_u16x8x4 = vld4q_u16((uint16_t const *)(data_ptr + idx * 4));
-            float16x8_t data_f16x8 = vreinterpretq_f16_u16(loaded_u16x8x4.val[0]);
-            float32x4_t low_f32x4 = vcvt_f32_f16(vget_low_f16(data_f16x8));
-            float32x4_t high_f32x4 = vcvt_f32_f16(vget_high_f16(data_f16x8));
-            sum_f32x4 = vaddq_f32(sum_f32x4, low_f32x4);
-            sum_f32x4 = vaddq_f32(sum_f32x4, high_f32x4);
-            sumsq_f32x4 = vfmaq_f32(sumsq_f32x4, low_f32x4, low_f32x4);
-            sumsq_f32x4 = vfmaq_f32(sumsq_f32x4, high_f32x4, high_f32x4);
-        }
-    }
-
-    // Scalar tail for remaining elements
-    nk_f32_t sum = vaddvq_f32(sum_f32x4);
-    nk_f32_t sumsq = vaddvq_f32(sumsq_f32x4);
-    for (; idx < count; ++idx) {
-        nk_f32_t value_f32;
-        nk_f16_to_f32_serial((nk_f16_t const *)(data_ptr + idx * stride_elements), &value_f32);
-        sum += value_f32, sumsq += value_f32 * value_f32;
-    }
-    *sum_ptr = sum, *sumsq_ptr = sumsq;
-}
-
-NK_PUBLIC void nk_reduce_moments_f16_neonhalf(                         //
-    nk_f16_t const *data_ptr, nk_size_t count, nk_size_t stride_bytes, //
-    nk_f32_t *sum_ptr, nk_f32_t *sumsq_ptr) {
-    nk_size_t stride_elements = stride_bytes / sizeof(nk_f16_t);
-    int aligned = (stride_bytes % sizeof(nk_f16_t) == 0);
-    if (count == 0) *sum_ptr = 0, *sumsq_ptr = 0;
-    else if (!aligned) nk_reduce_moments_f16_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr);
-    else if (count > (nk_size_t)(NK_U16_MAX + 1) * 8) {
-        nk_size_t left_count = count / 2;
-        nk_f32_t left_sum_value, left_sumsq_value, right_sum_value, right_sumsq_value;
-        nk_reduce_moments_f16_neonhalf(data_ptr, left_count, stride_bytes, &left_sum_value, &left_sumsq_value);
-        nk_reduce_moments_f16_neonhalf(data_ptr + left_count * stride_elements, count - left_count, stride_bytes,
-                                       &right_sum_value, &right_sumsq_value);
-        *sum_ptr = left_sum_value + right_sum_value, *sumsq_ptr = left_sumsq_value + right_sumsq_value;
-    }
-    else if (stride_elements == 1) nk_reduce_moments_f16_neonhalf_contiguous_(data_ptr, count, sum_ptr, sumsq_ptr);
-    else if (stride_elements <= 4)
-        nk_reduce_moments_f16_neonhalf_strided_(data_ptr, count, stride_elements, sum_ptr, sumsq_ptr);
-    else nk_reduce_moments_f16_serial(data_ptr, count, stride_bytes, sum_ptr, sumsq_ptr);
-}
-
-#if defined(__clang__)
-#pragma clang attribute pop
-#elif defined(__GNUC__)
-#pragma GCC pop_options
-#endif
-
-#if defined(__cplusplus)
-} // extern "C"
-#endif
-
-#endif // NK_TARGET_NEONHALF
-#endif // NK_TARGET_ARM_
-#endif // NK_REDUCE_NEONHALF_H

package/include/numkong/spatial/neonhalf.h

@@ -1,118 +0,0 @@
-/**
- *  @brief SIMD-accelerated Spatial Similarity Measures for NEON FP16.
- *  @file include/numkong/spatial/neonhalf.h
- *  @author Ash Vardanian
- *  @date December 27, 2025
- *
- *  @sa include/numkong/spatial.h
- *
- *  @section spatial_neonhalf_instructions ARM NEON FP16 Instructions (ARMv8.2-FP16)
- *
- *      Intrinsic                   Instruction                     Latency     Throughput
- *                                                                              A76         M4+/V1+/Oryon
- *      vfmaq_f16                   FMLA (V.8H, V.8H, V.8H)         4cy         2/cy        4/cy
- *      vcvt_f32_f16                FCVTL (V.4S, V.4H)              3cy         2/cy        4/cy
- *      vld1q_f16                   LD1 (V.8H)                      4cy         2/cy        3/cy
- *      vsubq_f16                   FSUB (V.8H, V.8H, V.8H)         2cy         2/cy        4/cy
- *      vaddvq_f32                  FADDP+FADDP (V.4S)              4cy         1/cy        2/cy
- *
- *  The ARMv8.2-FP16 extension enables native half-precision arithmetic, doubling the element count
- *  per vector register (8x F16 vs 4x F32). For spatial distance computations like L2 and angular
- *  distance, this halves memory bandwidth requirements.
- *
- *  Inputs are widened from F16 to F32 for accumulation via FCVTL to preserve numerical precision
- *  during the squared difference summation. The subtraction and FMA operations use F32 precision
- *  in the accumulator to avoid catastrophic cancellation in distance computations.
- */
-#ifndef NK_SPATIAL_NEONHALF_H
-#define NK_SPATIAL_NEONHALF_H
-
-#if NK_TARGET_ARM_
-#if NK_TARGET_NEONHALF
-
-#include "numkong/types.h"
-#include "numkong/cast/serial.h"  // `nk_partial_load_b16x4_serial_`
-#include "numkong/spatial/neon.h" // `nk_angular_normalize_f32_neon_`, `nk_f32_sqrt_neon`
-
-#if defined(__cplusplus)
-extern "C" {
-#endif
-
-#if defined(__clang__)
-#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
-#elif defined(__GNUC__)
-#pragma GCC push_options
-#pragma GCC target("arch=armv8.2-a+simd+fp16")
-#endif
-
-NK_PUBLIC void nk_sqeuclidean_f16_neonhalf(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *result) {
-    float32x4_t a_f32x4, b_f32x4;
-    float32x4_t distance_sq_f32x4 = vdupq_n_f32(0);
-
-nk_sqeuclidean_f16_neonhalf_cycle:
-    if (n < 4) {
-        nk_b64_vec_t a_vec, b_vec;
-        nk_partial_load_b16x4_serial_(a, &a_vec, n);
-        nk_partial_load_b16x4_serial_(b, &b_vec, n);
-        a_f32x4 = vcvt_f32_f16(vreinterpret_f16_u16(a_vec.u16x4));
-        b_f32x4 = vcvt_f32_f16(vreinterpret_f16_u16(b_vec.u16x4));
-        n = 0;
-    }
-    else {
-        a_f32x4 = vcvt_f32_f16(vld1_f16((nk_f16_for_arm_simd_t const *)a));
-        b_f32x4 = vcvt_f32_f16(vld1_f16((nk_f16_for_arm_simd_t const *)b));
-        n -= 4, a += 4, b += 4;
-    }
-    float32x4_t diff_f32x4 = vsubq_f32(a_f32x4, b_f32x4);
-    distance_sq_f32x4 = vfmaq_f32(distance_sq_f32x4, diff_f32x4, diff_f32x4);
-    if (n) goto nk_sqeuclidean_f16_neonhalf_cycle;
-
-    *result = vaddvq_f32(distance_sq_f32x4);
-}
-NK_PUBLIC void nk_euclidean_f16_neonhalf(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *result) {
-    nk_sqeuclidean_f16_neonhalf(a, b, n, result);
-    *result = nk_f32_sqrt_neon(*result);
-}
-
-NK_PUBLIC void nk_angular_f16_neonhalf(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t *result) {
-    float32x4_t dot_product_f32x4 = vdupq_n_f32(0), a_norm_sq_f32x4 = vdupq_n_f32(0), b_norm_sq_f32x4 = vdupq_n_f32(0);
-    float32x4_t a_f32x4, b_f32x4;
-
-nk_angular_f16_neonhalf_cycle:
-    if (n < 4) {
-        nk_b64_vec_t a_vec, b_vec;
-        nk_partial_load_b16x4_serial_(a, &a_vec, n);
-        nk_partial_load_b16x4_serial_(b, &b_vec, n);
-        a_f32x4 = vcvt_f32_f16(vreinterpret_f16_u16(a_vec.u16x4));
-        b_f32x4 = vcvt_f32_f16(vreinterpret_f16_u16(b_vec.u16x4));
-        n = 0;
-    }
-    else {
-        a_f32x4 = vcvt_f32_f16(vld1_f16((nk_f16_for_arm_simd_t const *)a));
-        b_f32x4 = vcvt_f32_f16(vld1_f16((nk_f16_for_arm_simd_t const *)b));
-        n -= 4, a += 4, b += 4;
-    }
-    dot_product_f32x4 = vfmaq_f32(dot_product_f32x4, a_f32x4, b_f32x4);
-    a_norm_sq_f32x4 = vfmaq_f32(a_norm_sq_f32x4, a_f32x4, a_f32x4);
-    b_norm_sq_f32x4 = vfmaq_f32(b_norm_sq_f32x4, b_f32x4, b_f32x4);
-    if (n) goto nk_angular_f16_neonhalf_cycle;
-
-    nk_f32_t dot_product_f32 = vaddvq_f32(dot_product_f32x4);
-    nk_f32_t a_norm_sq_f32 = vaddvq_f32(a_norm_sq_f32x4);
-    nk_f32_t b_norm_sq_f32 = vaddvq_f32(b_norm_sq_f32x4);
-    *result = nk_angular_normalize_f32_neon_(dot_product_f32, a_norm_sq_f32, b_norm_sq_f32);
-}
-
-#if defined(__clang__)
-#pragma clang attribute pop
-#elif defined(__GNUC__)
-#pragma GCC pop_options
-#endif
-
-#if defined(__cplusplus)
-} // extern "C"
-#endif
-
-#endif // NK_TARGET_NEONHALF
-#endif // NK_TARGET_ARM_
-#endif // NK_SPATIAL_NEONHALF_H

package/include/numkong/spatial/sapphire.h

@@ -1,343 +0,0 @@
-/**
- *  @brief SIMD-accelerated Spatial Similarity Measures for Sapphire Rapids.
- *  @file include/numkong/spatial/sapphire.h
- *  @author Ash Vardanian
- *  @date December 27, 2025
- *
- *  @sa include/numkong/spatial.h
- *
- *  Sapphire Rapids adds native FP16 support via AVX-512 FP16 extension.
- *  For e4m3 L2 distance, we can leverage F16 for the subtraction step:
- *  - e4m3 differences fit in F16 (max |a−b| = 896 < 65504)
- *  - But squared differences overflow F16 (896² = 802816 > 65504)
- *  - So: subtract in F16, convert to F32, then square and accumulate
- *
- *  For e2m3/e3m2 L2 distance, squared differences fit in FP16:
- *  - E2M3: max |a−b| = 15, max (a−b)² = 225 < 65504, flush cadence = 4 (conservative for uniformity)
- *  - E3M2: max |a−b| = 56, max (a−b)² = 3136 < 65504, flush cadence = 4
- *  So the entire sub+square+accumulate stays in FP16 with periodic F32 flush.
- *
- *  @section spatial_sapphire_instructions Relevant Instructions
- *
- *      Intrinsic                   Instruction                     Sapphire    Genoa
- *      _mm256_sub_ph               VSUBPH (YMM, YMM, YMM)          4cy @ p05   3cy @ p01
- *      _mm512_cvtph_ps             VCVTPH2PS (ZMM, YMM)            5cy @ p05   5cy @ p01
- *      _mm512_fmadd_ps             VFMADD (ZMM, ZMM, ZMM)          4cy @ p05   4cy @ p01
- *      _mm512_reduce_add_ps        (pseudo: VHADDPS chain)         ~8cy        ~8cy
- *      _mm_maskz_loadu_epi8        VMOVDQU8 (XMM {K}, M128)        7cy @ p23   7cy @ p23
- */
-#ifndef NK_SPATIAL_SAPPHIRE_H
-#define NK_SPATIAL_SAPPHIRE_H
-
-#if NK_TARGET_X86_
-#if NK_TARGET_SAPPHIRE
-
-#include "numkong/types.h"
-#include "numkong/cast/sapphire.h"   // `nk_e4m3x16_to_f16x16_sapphire_`
-#include "numkong/dot/sapphire.h"    // `nk_e2m3x32_to_f16x32_sapphire_`, `nk_flush_f16_to_f32_sapphire_`
-#include "numkong/spatial/haswell.h" // `nk_angular_normalize_f32_haswell_`, `nk_f32_sqrt_haswell`
-
-#if defined(__cplusplus)
-extern "C" {
-#endif
-
-#if defined(__clang__)
-#pragma clang attribute push(                                                                        \
-    __attribute__((target("avx2,avx512f,avx512vl,avx512bw,avx512dq,avx512fp16,f16c,fma,bmi,bmi2"))), \
-    apply_to = function)
-#elif defined(__GNUC__)
-#pragma GCC push_options
-#pragma GCC target("avx2", "avx512f", "avx512vl", "avx512bw", "avx512dq", "avx512fp16", "f16c", "fma", "bmi", "bmi2")
-#endif
-
-NK_PUBLIC void nk_sqeuclidean_e4m3_sapphire(nk_e4m3_t const *a_scalars, nk_e4m3_t const *b_scalars,
-                                            nk_size_t count_scalars, nk_f32_t *result) {
-    __m512 sum_f32x16 = _mm512_setzero_ps();
-
-    while (count_scalars > 0) {
-        nk_size_t const n = count_scalars < 16 ? count_scalars : 16;
-        __mmask16 const mask = (__mmask16)_bzhi_u32(0xFFFF, n);
-        __m128i a_e4m3x16 = _mm_maskz_loadu_epi8(mask, a_scalars);
-        __m128i b_e4m3x16 = _mm_maskz_loadu_epi8(mask, b_scalars);
-
-        // Convert e4m3 → f16
-        __m256h a_f16x16 = nk_e4m3x16_to_f16x16_sapphire_(a_e4m3x16);
-        __m256h b_f16x16 = nk_e4m3x16_to_f16x16_sapphire_(b_e4m3x16);
-
-        // Subtract in F16 − differences fit (max 896 < 65504)
-        __m256h diff_f16x16 = _mm256_sub_ph(a_f16x16, b_f16x16);
-
-        // Convert to F32 before squaring (896² = 802816 overflows F16!)
-        __m512 diff_f32x16 = _mm512_cvtph_ps(_mm256_castph_si256(diff_f16x16));
-
-        // Square and accumulate in F32
-        sum_f32x16 = _mm512_fmadd_ps(diff_f32x16, diff_f32x16, sum_f32x16);
-        a_scalars += n, b_scalars += n, count_scalars -= n;
-    }
-
-    *result = _mm512_reduce_add_ps(sum_f32x16);
-}
-
-NK_PUBLIC void nk_euclidean_e4m3_sapphire(nk_e4m3_t const *a_scalars, nk_e4m3_t const *b_scalars,
-                                          nk_size_t count_scalars, nk_f32_t *result) {
-    nk_sqeuclidean_e4m3_sapphire(a_scalars, b_scalars, count_scalars, result);
-    *result = nk_f32_sqrt_haswell(*result);
-}
-
-NK_PUBLIC void nk_sqeuclidean_e2m3_sapphire(nk_e2m3_t const *a_scalars, nk_e2m3_t const *b_scalars,
-                                            nk_size_t count_scalars, nk_f32_t *result) {
-    __m512 sum_f32x16 = _mm512_setzero_ps();
-
-    // Main loop: 4-way unrolled, 128 elements per flush
-    while (count_scalars >= 128) {
-        __m512h acc_f16x32 = _mm512_setzero_ph();
-        __m512h a_f16x32, b_f16x32, diff_f16x32;
-        // Iteration 1
-        a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars));
-        b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars));
-        diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
-        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
-        // Iteration 2
-        a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 32));
-        b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 32));
-        diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
-        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
-        // Iteration 3
-        a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 64));
-        b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 64));
-        diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
-        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
-        // Iteration 4
-        a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 96));
-        b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 96));
-        diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
-        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
-        // Flush to F32
-        sum_f32x16 = nk_flush_f16_to_f32_sapphire_(acc_f16x32, sum_f32x16);
-        a_scalars += 128, b_scalars += 128, count_scalars -= 128;
-    }
-
-    // Tail: remaining 0–127 elements, 32 at a time via masked loads
-    __m512h acc_f16x32 = _mm512_setzero_ph();
-    while (count_scalars > 0) {
-        nk_size_t const n = count_scalars < 32 ? count_scalars : 32;
-        __mmask32 const mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
-        __m512h a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_maskz_loadu_epi8(mask, a_scalars));
-        __m512h b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_maskz_loadu_epi8(mask, b_scalars));
-        __m512h diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
-        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
-        a_scalars += n, b_scalars += n, count_scalars -= n;
-    }
-    sum_f32x16 = nk_flush_f16_to_f32_sapphire_(acc_f16x32, sum_f32x16);
-
-    *result = nk_reduce_add_f32x16_skylake_(sum_f32x16);
-}
-
-NK_PUBLIC void nk_sqeuclidean_e3m2_sapphire(nk_e3m2_t const *a_scalars, nk_e3m2_t const *b_scalars,
-                                            nk_size_t count_scalars, nk_f32_t *result) {
-    __m512 sum_f32x16 = _mm512_setzero_ps();
-
-    // Main loop: 4-way unrolled, 128 elements per flush
-    while (count_scalars >= 128) {
-        __m512h acc_f16x32 = _mm512_setzero_ph();
-        __m512h a_f16x32, b_f16x32, diff_f16x32;
-        // Iteration 1
-        a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars));
-        b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars));
-        diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
-        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
-        // Iteration 2
-        a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 32));
-        b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 32));
-        diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
-        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
-        // Iteration 3
-        a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 64));
-        b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 64));
-        diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
-        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
-        // Iteration 4
-        a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 96));
-        b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 96));
-        diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
-        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
-        // Flush to F32
-        sum_f32x16 = nk_flush_f16_to_f32_sapphire_(acc_f16x32, sum_f32x16);
-        a_scalars += 128, b_scalars += 128, count_scalars -= 128;
-    }
-
-    // Tail: remaining 0–127 elements, 32 at a time via masked loads
-    __m512h acc_f16x32 = _mm512_setzero_ph();
-    while (count_scalars > 0) {
-        nk_size_t const n = count_scalars < 32 ? count_scalars : 32;
-        __mmask32 const mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
-        __m512h a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_maskz_loadu_epi8(mask, a_scalars));
-        __m512h b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_maskz_loadu_epi8(mask, b_scalars));
-        __m512h diff_f16x32 = _mm512_sub_ph(a_f16x32, b_f16x32);
-        acc_f16x32 = _mm512_fmadd_ph(diff_f16x32, diff_f16x32, acc_f16x32);
-        a_scalars += n, b_scalars += n, count_scalars -= n;
-    }
-    sum_f32x16 = nk_flush_f16_to_f32_sapphire_(acc_f16x32, sum_f32x16);
-
-    *result = nk_reduce_add_f32x16_skylake_(sum_f32x16);
-}
-
-NK_PUBLIC void nk_euclidean_e2m3_sapphire(nk_e2m3_t const *a_scalars, nk_e2m3_t const *b_scalars,
-                                          nk_size_t count_scalars, nk_f32_t *result) {
-    nk_sqeuclidean_e2m3_sapphire(a_scalars, b_scalars, count_scalars, result);
-    *result = nk_f32_sqrt_haswell(*result);
-}
-
-NK_PUBLIC void nk_euclidean_e3m2_sapphire(nk_e3m2_t const *a_scalars, nk_e3m2_t const *b_scalars,
-                                          nk_size_t count_scalars, nk_f32_t *result) {
-    nk_sqeuclidean_e3m2_sapphire(a_scalars, b_scalars, count_scalars, result);
-    *result = nk_f32_sqrt_haswell(*result);
-}
-
-NK_PUBLIC void nk_angular_e2m3_sapphire(nk_e2m3_t const *a_scalars, nk_e2m3_t const *b_scalars, nk_size_t count_scalars,
-                                        nk_f32_t *result) {
-    __m512 sum_dot_f32x16 = _mm512_setzero_ps();
-    __m512 sum_a_f32x16 = _mm512_setzero_ps();
-    __m512 sum_b_f32x16 = _mm512_setzero_ps();
-
-    // Main loop: 4-way unrolled, 128 elements per flush
-    while (count_scalars >= 128) {
-        __m512h dot_acc = _mm512_setzero_ph();
-        __m512h a_norm_acc = _mm512_setzero_ph();
-        __m512h b_norm_acc = _mm512_setzero_ph();
-        __m512h a_f16x32, b_f16x32;
-        // Iteration 1
-        a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars));
-        b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars));
-        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
-        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
-        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
-        // Iteration 2
-        a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 32));
-        b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 32));
-        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
-        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
-        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
-        // Iteration 3
-        a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 64));
-        b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 64));
-        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
-        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
-        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
-        // Iteration 4
-        a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 96));
-        b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 96));
-        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
-        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
-        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
-        // Flush to F32
-        sum_dot_f32x16 = nk_flush_f16_to_f32_sapphire_(dot_acc, sum_dot_f32x16);
-        sum_a_f32x16 = nk_flush_f16_to_f32_sapphire_(a_norm_acc, sum_a_f32x16);
-        sum_b_f32x16 = nk_flush_f16_to_f32_sapphire_(b_norm_acc, sum_b_f32x16);
-        a_scalars += 128, b_scalars += 128, count_scalars -= 128;
-    }
-
-    // Tail: remaining 0–127 elements, 32 at a time via masked loads
-    __m512h dot_acc = _mm512_setzero_ph();
-    __m512h a_norm_acc = _mm512_setzero_ph();
-    __m512h b_norm_acc = _mm512_setzero_ph();
-    while (count_scalars > 0) {
-        nk_size_t const n = count_scalars < 32 ? count_scalars : 32;
-        __mmask32 const mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
-        __m512h a_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_maskz_loadu_epi8(mask, a_scalars));
-        __m512h b_f16x32 = nk_e2m3x32_to_f16x32_sapphire_(_mm256_maskz_loadu_epi8(mask, b_scalars));
-        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
-        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
-        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
-        a_scalars += n, b_scalars += n, count_scalars -= n;
-    }
-    sum_dot_f32x16 = nk_flush_f16_to_f32_sapphire_(dot_acc, sum_dot_f32x16);
-    sum_a_f32x16 = nk_flush_f16_to_f32_sapphire_(a_norm_acc, sum_a_f32x16);
-    sum_b_f32x16 = nk_flush_f16_to_f32_sapphire_(b_norm_acc, sum_b_f32x16);
-
-    nk_f32_t dot_f32 = nk_reduce_add_f32x16_skylake_(sum_dot_f32x16);
-    nk_f32_t a_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(sum_a_f32x16);
-    nk_f32_t b_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(sum_b_f32x16);
-    *result = nk_angular_normalize_f32_haswell_(dot_f32, a_norm_sq_f32, b_norm_sq_f32);
-}
-
-NK_PUBLIC void nk_angular_e3m2_sapphire(nk_e3m2_t const *a_scalars, nk_e3m2_t const *b_scalars, nk_size_t count_scalars,
-                                        nk_f32_t *result) {
-    __m512 sum_dot_f32x16 = _mm512_setzero_ps();
-    __m512 sum_a_f32x16 = _mm512_setzero_ps();
-    __m512 sum_b_f32x16 = _mm512_setzero_ps();
-
-    // Main loop: 4-way unrolled, 128 elements per flush
-    while (count_scalars >= 128) {
-        __m512h dot_acc = _mm512_setzero_ph();
-        __m512h a_norm_acc = _mm512_setzero_ph();
-        __m512h b_norm_acc = _mm512_setzero_ph();
-        __m512h a_f16x32, b_f16x32;
-        // Iteration 1
-        a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars));
-        b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars));
-        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
-        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
-        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
-        // Iteration 2
-        a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 32));
-        b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 32));
-        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
-        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
-        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
-        // Iteration 3
-        a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 64));
-        b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 64));
-        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
-        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
-        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
-        // Iteration 4
-        a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(a_scalars + 96));
-        b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_loadu_epi8(b_scalars + 96));
-        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
-        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
-        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
-        // Flush to F32
-        sum_dot_f32x16 = nk_flush_f16_to_f32_sapphire_(dot_acc, sum_dot_f32x16);
-        sum_a_f32x16 = nk_flush_f16_to_f32_sapphire_(a_norm_acc, sum_a_f32x16);
-        sum_b_f32x16 = nk_flush_f16_to_f32_sapphire_(b_norm_acc, sum_b_f32x16);
-        a_scalars += 128, b_scalars += 128, count_scalars -= 128;
-    }
-
-    // Tail: remaining 0–127 elements, 32 at a time via masked loads
-    __m512h dot_acc = _mm512_setzero_ph();
-    __m512h a_norm_acc = _mm512_setzero_ph();
-    __m512h b_norm_acc = _mm512_setzero_ph();
-    while (count_scalars > 0) {
-        nk_size_t const n = count_scalars < 32 ? count_scalars : 32;
-        __mmask32 const mask = (__mmask32)_bzhi_u32(0xFFFFFFFF, n);
-        __m512h a_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_maskz_loadu_epi8(mask, a_scalars));
-        __m512h b_f16x32 = nk_e3m2x32_to_f16x32_sapphire_(_mm256_maskz_loadu_epi8(mask, b_scalars));
-        dot_acc = _mm512_fmadd_ph(a_f16x32, b_f16x32, dot_acc);
-        a_norm_acc = _mm512_fmadd_ph(a_f16x32, a_f16x32, a_norm_acc);
-        b_norm_acc = _mm512_fmadd_ph(b_f16x32, b_f16x32, b_norm_acc);
-        a_scalars += n, b_scalars += n, count_scalars -= n;
-    }
-    sum_dot_f32x16 = nk_flush_f16_to_f32_sapphire_(dot_acc, sum_dot_f32x16);
-    sum_a_f32x16 = nk_flush_f16_to_f32_sapphire_(a_norm_acc, sum_a_f32x16);
-    sum_b_f32x16 = nk_flush_f16_to_f32_sapphire_(b_norm_acc, sum_b_f32x16);
-
-    nk_f32_t dot_f32 = nk_reduce_add_f32x16_skylake_(sum_dot_f32x16);
-    nk_f32_t a_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(sum_a_f32x16);
-    nk_f32_t b_norm_sq_f32 = nk_reduce_add_f32x16_skylake_(sum_b_f32x16);
-    *result = nk_angular_normalize_f32_haswell_(dot_f32, a_norm_sq_f32, b_norm_sq_f32);
-}
-
-#if defined(__clang__)
-#pragma clang attribute pop
-#elif defined(__GNUC__)
-#pragma GCC pop_options
-#endif
-
-#if defined(__cplusplus)
-} // extern "C"
-#endif
-
-#endif // NK_TARGET_SAPPHIRE
-#endif // NK_TARGET_X86_
-#endif // NK_SPATIAL_SAPPHIRE_H

package/include/numkong/spatials/neonhalf.h

@@ -1,58 +0,0 @@
-/**
- *  @brief Batched Spatial Distances for NEON FP16 (Half-Precision).
- *  @file include/numkong/spatials/neonhalf.h
- *  @author Ash Vardanian
- *  @date February 23, 2026
- *
- *  @sa include/numkong/spatials.h
- */
-#ifndef NK_SPATIALS_NEONHALF_H
-#define NK_SPATIALS_NEONHALF_H
-
-#if NK_TARGET_ARM_
-#if NK_TARGET_NEONHALF
-
-#include "numkong/spatial/neon.h"
-#include "numkong/dots/neonhalf.h"
-
-#if defined(__cplusplus)
-extern "C" {
-#endif
-
-#if defined(__clang__)
-#pragma clang attribute push(__attribute__((target("arch=armv8.2-a+simd+fp16"))), apply_to = function)
-#elif defined(__GNUC__)
-#pragma GCC push_options
-#pragma GCC target("arch=armv8.2-a+simd+fp16")
-#endif
-
-nk_define_cross_normalized_packed_(angular, f16, neonhalf, f16, f16, f32, /*norm_value_type=*/f32, f32, nk_b128_vec_t,
-                                   nk_dots_packed_f16_neonhalf, nk_angular_through_f32_from_dot_neon_,
-                                   nk_dots_reduce_sumsq_f16_, nk_load_b128_neon_, nk_partial_load_b32x4_serial_,
-                                   nk_store_b128_neon_, nk_partial_store_b32x4_serial_, 1)
-nk_define_cross_normalized_packed_(euclidean, f16, neonhalf, f16, f16, f32, /*norm_value_type=*/f32, f32, nk_b128_vec_t,
-                                   nk_dots_packed_f16_neonhalf, nk_euclidean_through_f32_from_dot_neon_,
-                                   nk_dots_reduce_sumsq_f16_, nk_load_b128_neon_, nk_partial_load_b32x4_serial_,
-                                   nk_store_b128_neon_, nk_partial_store_b32x4_serial_, 1)
-nk_define_cross_normalized_symmetric_(angular, f16, neonhalf, f16, f32, /*norm_value_type=*/f32, f32, nk_b128_vec_t,
-                                      nk_dots_symmetric_f16_neonhalf, nk_angular_through_f32_from_dot_neon_,
-                                      nk_dots_reduce_sumsq_f16_, nk_load_b128_neon_, nk_partial_load_b32x4_serial_,
-                                      nk_store_b128_neon_, nk_partial_store_b32x4_serial_, 1)
-nk_define_cross_normalized_symmetric_(euclidean, f16, neonhalf, f16, f32, /*norm_value_type=*/f32, f32, nk_b128_vec_t,
-                                      nk_dots_symmetric_f16_neonhalf, nk_euclidean_through_f32_from_dot_neon_,
-                                      nk_dots_reduce_sumsq_f16_, nk_load_b128_neon_, nk_partial_load_b32x4_serial_,
-                                      nk_store_b128_neon_, nk_partial_store_b32x4_serial_, 1)
-
-#if defined(__clang__)
-#pragma clang attribute pop
-#elif defined(__GNUC__)
-#pragma GCC pop_options
-#endif
-
-#if defined(__cplusplus)
-} // extern "C"
-#endif
-
-#endif // NK_TARGET_NEONHALF
-#endif // NK_TARGET_ARM_
-#endif // NK_SPATIALS_NEONHALF_H