numkong 7.0.0 → 7.4.2

package/include/numkong/each.h

@@ -6,32 +6,34 @@
  *
  *  Contains following element-wise operations:
  *
- *  - Scale (Multiply) with shift: result[i] = alpha * a[i] + beta
+ *  - Scale (Multiply) with shift: result[i] = α·a[i] + β
  *  - Sum (Add): result[i] = a[i] + b[i]
- *  - Blend: result[i] = alpha * a[i] + beta * b[i]
- *  - FMA (Fused Multiply-Add): result[i] = alpha * a[i] * b[i] + beta * c[i]
+ *  - Blend: result[i] = α·a[i] + β·b[i]
+ *  - FMA (Fused Multiply-Add): result[i] = α·a[i]·b[i] + β·c[i]
  *
  *  Beyond their obvious usecases, those can be reused for vector-scalar math and other operations:
  *
- *  - Scale with beta = 0 for a pure multiply.
- *  - Sum is equivalent to WSum with alpha = beta = 1.
- *  - Average is WSum with alpha = beta = 0.5.
- *  - Elementwise multiply is FMA with beta = 0.
+ *  - Scale with β = 0 for a pure multiply.
+ *  - Sum is equivalent to WSum with α = β = 1.
+ *  - Average is WSum with α = β = 0.5.
+ *  - Elementwise multiply is FMA with β = 0.
  *
  *  For dtypes:
  *
- *  - f64: 64-bit IEEE floating point numbers × 64-bit scales
- *  - f32: 32-bit IEEE floating point numbers × 32-bit scales
- *  - f16: 16-bit IEEE floating point numbers × 32-bit scales
- *  - bf16: 16-bit brain floating point numbers × 32-bit scales
- *  - e4m3: 8-bit e4m3 floating point numbers × 32-bit scales
- *  - e5m2: 8-bit e5m2 floating point numbers × 32-bit scales
- *  - e2m3: 8-bit e2m3 floating point numbers (MX) × 32-bit scales
- *  - e3m2: 8-bit e3m2 floating point numbers (MX) × 32-bit scales
- *  - i8/u8: 8-bit signed and unsigned integers × 32-bit scales
- *  - i16/u16: 16-bit signed and unsigned integers × 32-bit scales
- *  - i32/u32: 32-bit signed and unsigned integers × 64-bit scales
- *  - i64/u64: 64-bit signed and unsigned integers × 64-bit scales
+ *  - f64c: 64-bit complex × 64-bit complex scales
+ *  - f32c: 32-bit complex × 32-bit complex scales
+ *  - f64: 64-bit IEEE floating point × 64-bit scales
+ *  - f32: 32-bit IEEE floating point × 32-bit scales
+ *  - f16: 16-bit IEEE floating point × 32-bit scales
+ *  - bf16: 16-bit brain floating point × 32-bit scales
+ *  - e4m3: 8-bit e4m3 floating point × 32-bit scales
+ *  - e5m2: 8-bit e5m2 floating point × 32-bit scales
+ *  - e2m3: 8-bit e2m3 floating point (MX) × 32-bit scales
+ *  - e3m2: 8-bit e3m2 floating point (MX) × 32-bit scales
+ *  - i8/u8: 8-bit integers × 32-bit scales
+ *  - i16/u16: 16-bit integers × 32-bit scales
+ *  - i32/u32: 32-bit integers × 64-bit scales
+ *  - i64/u64: 64-bit integers × 64-bit scales
  *
  *  For hardware architectures:
  *
@@ -55,13 +57,13 @@
  *  Saturating integer adds (VPADDSW/VPADDUSW) provide overflow protection for i16/u16 sums without
  *  branching. FMA (VFMADD231PS) is the workhorse for scale (alpha*x+beta) and blend (alpha*a+beta*b).
  *
- *      Intrinsic               Instruction                     Ice         Genoa
- *      _mm512_cvtph_ps         VCVTPH2PS (ZMM, YMM)            7c @ p0+p5  6c @ p12+p23
- *      _mm512_cvtps_ph         VCVTPS2PH (YMM, ZMM, I8)        7c @ p0+p5  7c @ p12+p23
- *      _mm256_adds_epi16       VPADDSW (YMM, YMM, YMM)         1c @ p01    N/A
- *      _mm256_adds_epu16       VPADDUSW (YMM, YMM, YMM)        1c @ p01    N/A
- *      _mm512_fpclass_ps_mask  VFPCLASSPS (K, ZMM, I8)         3c @ p5     5c @ p01
- *      _mm256_fmadd_ps         VFMADD231PS (YMM, YMM, YMM)     4c @ p01    4c @ p01
+ *      Intrinsic               Instruction                  Icelake      Genoa
+ *      _mm512_cvtph_ps         VCVTPH2PS (ZMM, YMM)         7cy @ p0+p5  6cy @ p12+p23
+ *      _mm512_cvtps_ph         VCVTPS2PH (YMM, ZMM, I8)     7cy @ p0+p5  7cy @ p12+p23
+ *      _mm256_adds_epi16       VPADDSW (YMM, YMM, YMM)      1cy @ p01    n/a
+ *      _mm256_adds_epu16       VPADDUSW (YMM, YMM, YMM)     1cy @ p01    n/a
+ *      _mm512_fpclass_ps_mask  VFPCLASSPS (K, ZMM, I8)      3cy @ p5     5cy @ p01
+ *      _mm256_fmadd_ps         VFMADD231PS (YMM, YMM, YMM)  4cy @ p01    4cy @ p01
  *
  *  @section arm_instructions Relevant ARM NEON/SVE Instructions
  *
@@ -69,16 +71,16 @@
  *  vector throughput (8 elements per 128-bit register vs 4 for f32). Saturating adds (SQADD/UQADD)
  *  handle integer overflow. FMLA provides fused multiply-add for floating-point scale/blend/fma.
  *
- *      Intrinsic               Instruction     M1 Firestorm    Graviton 3      Graviton 4
- *      vfmaq_f32               FMLA.S (vec)    4c @ V0123      4c @ V0123      4c @ V0123
- *      vqaddq_s16              SQADD (vec)     3c @ V0123      2c @ V0123      2c @ V0123
- *      vqaddq_u16              UQADD (vec)     3c @ V0123      2c @ V0123      2c @ V0123
- *      vcvtq_f32_s32           SCVTF (vec)     3c @ V0123      3c @ V01        3c @ V01
- *      vcvtnq_s32_f32          FCVTNS (vec)    3c @ V0123      3c @ V01        3c @ V01
+ *      Intrinsic       Instruction   M1 Firestorm  Graviton 3   Graviton 4
+ *      vfmaq_f32       FMLA.S (vec)  4cy @ V0123   4cy @ V0123  4cy @ V0123
+ *      vqaddq_s16      SQADD (vec)   3cy @ V0123   2cy @ V0123  2cy @ V0123
+ *      vqaddq_u16      UQADD (vec)   3cy @ V0123   2cy @ V0123  2cy @ V0123
+ *      vcvtq_f32_s32   SCVTF (vec)   3cy @ V0123   3cy @ V01    3cy @ V01
+ *      vcvtnq_s32_f32  FCVTNS (vec)  3cy @ V0123   3cy @ V01    3cy @ V01
  *
  *  @section references References
  *
- *  - x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/
+ *  - x86 intrinsics: https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html
  *  - Arm intrinsics: https://developer.arm.com/architectures/instruction-sets/intrinsics/
  *
  */
@@ -651,6 +653,11 @@ NK_PUBLIC void nk_each_fma_f32c_neon(nk_f32c_t const *a, nk_f32c_t const *b, nk_
 /** @copydoc nk_each_fma_f64 */
 NK_PUBLIC void nk_each_fma_f64c_neon(nk_f64c_t const *a, nk_f64c_t const *b, nk_f64c_t const *c, nk_size_t n,
                                      nk_f64c_t const *alpha, nk_f64c_t const *beta, nk_f64c_t *result);
+
+/** @copydoc nk_each_sum_i8 */
+NK_PUBLIC void nk_each_sum_i8_neon(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_i8_t *result);
+/** @copydoc nk_each_sum_u8 */
+NK_PUBLIC void nk_each_sum_u8_neon(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_u8_t *result);
 #endif // NK_TARGET_NEON
 
 #if NK_TARGET_NEONBFDOT
@@ -680,10 +687,6 @@ NK_PUBLIC void nk_each_blend_f16_neonhalf(nk_f16_t const *a, nk_f16_t const *b,
 NK_PUBLIC void nk_each_fma_f16_neonhalf(nk_f16_t const *a, nk_f16_t const *b, nk_f16_t const *c, nk_size_t n,
                                         nk_f32_t const *alpha, nk_f32_t const *beta, nk_f16_t *result);
 
-/** @copydoc nk_each_sum_i8 */
-NK_PUBLIC void nk_each_sum_i8_neonhalf(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_i8_t *result);
-/** @copydoc nk_each_sum_u8 */
-NK_PUBLIC void nk_each_sum_u8_neonhalf(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_u8_t *result);
 /** @copydoc nk_each_scale_i8 */
 NK_PUBLIC void nk_each_scale_i8_neonhalf(nk_i8_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
                                          nk_i8_t *result);
@@ -696,13 +699,65 @@ NK_PUBLIC void nk_each_blend_i8_neonhalf(nk_i8_t const *a, nk_i8_t const *b, nk_
 /** @copydoc nk_each_blend_u8 */
 NK_PUBLIC void nk_each_blend_u8_neonhalf(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_f32_t const *alpha,
                                          nk_f32_t const *beta, nk_u8_t *result);
+#endif // NK_TARGET_NEONHALF
+
+#if NK_TARGET_V128RELAXED
+/** @copydoc nk_each_sum_f32 */
+NK_PUBLIC void nk_each_sum_f32_v128relaxed(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t *result);
+/** @copydoc nk_each_scale_f32 */
+NK_PUBLIC void nk_each_scale_f32_v128relaxed(nk_f32_t const *a, nk_size_t n, nk_f32_t const *alpha,
+                                             nk_f32_t const *beta, nk_f32_t *result);
+/** @copydoc nk_each_blend_f32 */
+NK_PUBLIC void nk_each_blend_f32_v128relaxed(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n, nk_f32_t const *alpha,
+                                             nk_f32_t const *beta, nk_f32_t *result);
+/** @copydoc nk_each_fma_f32 */
+NK_PUBLIC void nk_each_fma_f32_v128relaxed(nk_f32_t const *a, nk_f32_t const *b, nk_f32_t const *c, nk_size_t n,
+                                           nk_f32_t const *alpha, nk_f32_t const *beta, nk_f32_t *result);
+/** @copydoc nk_each_sum_f16 */
+NK_PUBLIC void nk_each_sum_f16_v128relaxed(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f16_t *result);
+/** @copydoc nk_each_scale_f16 */
+NK_PUBLIC void nk_each_scale_f16_v128relaxed(nk_f16_t const *a, nk_size_t n, nk_f32_t const *alpha,
+                                             nk_f32_t const *beta, nk_f16_t *result);
+/** @copydoc nk_each_blend_f16 */
+NK_PUBLIC void nk_each_blend_f16_v128relaxed(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n, nk_f32_t const *alpha,
+                                             nk_f32_t const *beta, nk_f16_t *result);
+/** @copydoc nk_each_fma_f16 */
+NK_PUBLIC void nk_each_fma_f16_v128relaxed(nk_f16_t const *a, nk_f16_t const *b, nk_f16_t const *c, nk_size_t n,
+                                           nk_f32_t const *alpha, nk_f32_t const *beta, nk_f16_t *result);
+/** @copydoc nk_each_sum_bf16 */
+NK_PUBLIC void nk_each_sum_bf16_v128relaxed(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n, nk_bf16_t *result);
+/** @copydoc nk_each_scale_bf16 */
+NK_PUBLIC void nk_each_scale_bf16_v128relaxed(nk_bf16_t const *a, nk_size_t n, nk_f32_t const *alpha,
+                                              nk_f32_t const *beta, nk_bf16_t *result);
+/** @copydoc nk_each_blend_bf16 */
+NK_PUBLIC void nk_each_blend_bf16_v128relaxed(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_t n,
+                                              nk_f32_t const *alpha, nk_f32_t const *beta, nk_bf16_t *result);
+/** @copydoc nk_each_fma_bf16 */
+NK_PUBLIC void nk_each_fma_bf16_v128relaxed(nk_bf16_t const *a, nk_bf16_t const *b, nk_bf16_t const *c, nk_size_t n,
+                                            nk_f32_t const *alpha, nk_f32_t const *beta, nk_bf16_t *result);
+/** @copydoc nk_each_sum_i8 */
+NK_PUBLIC void nk_each_sum_i8_v128relaxed(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_i8_t *result);
+/** @copydoc nk_each_scale_i8 */
+NK_PUBLIC void nk_each_scale_i8_v128relaxed(nk_i8_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
+                                            nk_i8_t *result);
+/** @copydoc nk_each_blend_i8 */
+NK_PUBLIC void nk_each_blend_i8_v128relaxed(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, nk_f32_t const *alpha,
+                                            nk_f32_t const *beta, nk_i8_t *result);
 /** @copydoc nk_each_fma_i8 */
-NK_PUBLIC void nk_each_fma_i8_neonhalf(nk_i8_t const *a, nk_i8_t const *b, nk_i8_t const *c, nk_size_t n,
-                                       nk_f32_t const *alpha, nk_f32_t const *beta, nk_i8_t *result);
+NK_PUBLIC void nk_each_fma_i8_v128relaxed(nk_i8_t const *a, nk_i8_t const *b, nk_i8_t const *c, nk_size_t n,
+                                          nk_f32_t const *alpha, nk_f32_t const *beta, nk_i8_t *result);
+/** @copydoc nk_each_sum_u8 */
+NK_PUBLIC void nk_each_sum_u8_v128relaxed(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_u8_t *result);
+/** @copydoc nk_each_scale_u8 */
+NK_PUBLIC void nk_each_scale_u8_v128relaxed(nk_u8_t const *a, nk_size_t n, nk_f32_t const *alpha, nk_f32_t const *beta,
+                                            nk_u8_t *result);
+/** @copydoc nk_each_blend_u8 */
+NK_PUBLIC void nk_each_blend_u8_v128relaxed(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_f32_t const *alpha,
+                                            nk_f32_t const *beta, nk_u8_t *result);
 /** @copydoc nk_each_fma_u8 */
-NK_PUBLIC void nk_each_fma_u8_neonhalf(nk_u8_t const *a, nk_u8_t const *b, nk_u8_t const *c, nk_size_t n,
-                                       nk_f32_t const *alpha, nk_f32_t const *beta, nk_u8_t *result);
-#endif // NK_TARGET_NEONHALF
+NK_PUBLIC void nk_each_fma_u8_v128relaxed(nk_u8_t const *a, nk_u8_t const *b, nk_u8_t const *c, nk_size_t n,
+                                          nk_f32_t const *alpha, nk_f32_t const *beta, nk_u8_t *result);
+#endif // NK_TARGET_V128RELAXED
 
 #if NK_TARGET_HASWELL
 /** @copydoc nk_each_scale_f64 */
@@ -1026,12 +1081,6 @@ NK_PUBLIC void nk_each_blend_i8_sapphire(nk_i8_t const *a, nk_i8_t const *b, nk_
 NK_PUBLIC void nk_each_blend_u8_sapphire(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, nk_f32_t const *alpha,
                                          nk_f32_t const *beta, nk_u8_t *result);
 
-/** @copydoc nk_each_fma_i8 */
-NK_PUBLIC void nk_each_fma_i8_sapphire(nk_i8_t const *a, nk_i8_t const *b, nk_i8_t const *c, nk_size_t n,
-                                       nk_f32_t const *alpha, nk_f32_t const *beta, nk_i8_t *result);
-/** @copydoc nk_each_fma_u8 */
-NK_PUBLIC void nk_each_fma_u8_sapphire(nk_u8_t const *a, nk_u8_t const *b, nk_u8_t const *c, nk_size_t n,
-                                       nk_f32_t const *alpha, nk_f32_t const *beta, nk_u8_t *result);
 #endif // NK_TARGET_SAPPHIRE
 
 #if NK_TARGET_RVV
@@ -1213,6 +1262,10 @@ NK_INTERNAL nk_dtype_t nk_each_scale_input_dtype(nk_dtype_t dtype) {
     case nk_u16_k: return nk_f32_k;
     case nk_i8_k: return nk_f32_k;
     case nk_u8_k: return nk_f32_k;
+    case nk_e4m3_k: return nk_f32_k;
+    case nk_e5m2_k: return nk_f32_k;
+    case nk_e2m3_k: return nk_f32_k;
+    case nk_e3m2_k: return nk_f32_k;
     default: return nk_dtype_unknown_k;
     }
 }
@@ -1230,6 +1283,7 @@ NK_INTERNAL nk_dtype_t nk_each_scale_input_dtype(nk_dtype_t dtype) {
 #include "numkong/each/icelake.h"
 #include "numkong/each/sapphire.h"
 #include "numkong/each/rvv.h"
+#include "numkong/each/v128relaxed.h"
 
 #if defined(__cplusplus)
 extern "C" {
@@ -1260,6 +1314,8 @@ NK_PUBLIC void nk_each_sum_f32(nk_f32_t const *a, nk_f32_t const *b, nk_size_t n
     nk_each_sum_f32_neon(a, b, n, r);
 #elif NK_TARGET_RVV
     nk_each_sum_f32_rvv(a, b, n, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_sum_f32_v128relaxed(a, b, n, r);
 #else
     nk_each_sum_f32_serial(a, b, n, r);
 #endif
@@ -1274,6 +1330,8 @@ NK_PUBLIC void nk_each_sum_bf16(nk_bf16_t const *a, nk_bf16_t const *b, nk_size_
     nk_each_sum_bf16_neonbfdot(a, b, n, r);
 #elif NK_TARGET_RVV
     nk_each_sum_bf16_rvv(a, b, n, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_sum_bf16_v128relaxed(a, b, n, r);
 #else
     nk_each_sum_bf16_serial(a, b, n, r);
 #endif
@@ -1288,6 +1346,8 @@ NK_PUBLIC void nk_each_sum_f16(nk_f16_t const *a, nk_f16_t const *b, nk_size_t n
     nk_each_sum_f16_neonhalf(a, b, n, r);
 #elif NK_TARGET_RVV
     nk_each_sum_f16_rvv(a, b, n, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_sum_f16_v128relaxed(a, b, n, r);
 #else
     nk_each_sum_f16_serial(a, b, n, r);
 #endif
@@ -1298,10 +1358,12 @@ NK_PUBLIC void nk_each_sum_i8(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n, n
     nk_each_sum_i8_icelake(a, b, n, r);
 #elif NK_TARGET_HASWELL
     nk_each_sum_i8_haswell(a, b, n, r);
-#elif NK_TARGET_NEONHALF
-    nk_each_sum_i8_neonhalf(a, b, n, r);
+#elif NK_TARGET_NEON
+    nk_each_sum_i8_neon(a, b, n, r);
 #elif NK_TARGET_RVV
     nk_each_sum_i8_rvv(a, b, n, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_sum_i8_v128relaxed(a, b, n, r);
 #else
     nk_each_sum_i8_serial(a, b, n, r);
 #endif
@@ -1312,10 +1374,12 @@ NK_PUBLIC void nk_each_sum_u8(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n, n
     nk_each_sum_u8_icelake(a, b, n, r);
 #elif NK_TARGET_HASWELL
     nk_each_sum_u8_haswell(a, b, n, r);
-#elif NK_TARGET_NEONHALF
-    nk_each_sum_u8_neonhalf(a, b, n, r);
+#elif NK_TARGET_NEON
+    nk_each_sum_u8_neon(a, b, n, r);
 #elif NK_TARGET_RVV
     nk_each_sum_u8_rvv(a, b, n, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_sum_u8_v128relaxed(a, b, n, r);
 #else
     nk_each_sum_u8_serial(a, b, n, r);
 #endif
@@ -1426,6 +1490,8 @@ NK_PUBLIC void nk_each_scale_f32(nk_f32_t const *a, nk_size_t n, nk_f32_t const
     nk_each_scale_f32_neon(a, n, alpha, beta, r);
 #elif NK_TARGET_RVV
     nk_each_scale_f32_rvv(a, n, alpha, beta, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_scale_f32_v128relaxed(a, n, alpha, beta, r);
 #else
     nk_each_scale_f32_serial(a, n, alpha, beta, r);
 #endif
@@ -1441,6 +1507,8 @@ NK_PUBLIC void nk_each_scale_bf16(nk_bf16_t const *a, nk_size_t n, nk_f32_t cons
     nk_each_scale_bf16_neonbfdot(a, n, alpha, beta, r);
 #elif NK_TARGET_RVV
     nk_each_scale_bf16_rvv(a, n, alpha, beta, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_scale_bf16_v128relaxed(a, n, alpha, beta, r);
 #else
     nk_each_scale_bf16_serial(a, n, alpha, beta, r);
 #endif
@@ -1456,6 +1524,8 @@ NK_PUBLIC void nk_each_scale_f16(nk_f16_t const *a, nk_size_t n, nk_f32_t const
     nk_each_scale_f16_neonhalf(a, n, alpha, beta, r);
 #elif NK_TARGET_RVV
     nk_each_scale_f16_rvv(a, n, alpha, beta, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_scale_f16_v128relaxed(a, n, alpha, beta, r);
 #else
     nk_each_scale_f16_serial(a, n, alpha, beta, r);
 #endif
@@ -1473,6 +1543,8 @@ NK_PUBLIC void nk_each_scale_i8(nk_i8_t const *a, nk_size_t n, nk_f32_t const *a
     nk_each_scale_i8_neonhalf(a, n, alpha, beta, r);
 #elif NK_TARGET_RVV
     nk_each_scale_i8_rvv(a, n, alpha, beta, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_scale_i8_v128relaxed(a, n, alpha, beta, r);
 #else
     nk_each_scale_i8_serial(a, n, alpha, beta, r);
 #endif
@@ -1490,6 +1562,8 @@ NK_PUBLIC void nk_each_scale_u8(nk_u8_t const *a, nk_size_t n, nk_f32_t const *a
     nk_each_scale_u8_neonhalf(a, n, alpha, beta, r);
 #elif NK_TARGET_RVV
     nk_each_scale_u8_rvv(a, n, alpha, beta, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_scale_u8_v128relaxed(a, n, alpha, beta, r);
 #else
     nk_each_scale_u8_serial(a, n, alpha, beta, r);
 #endif
@@ -1606,6 +1680,8 @@ NK_PUBLIC void nk_each_blend_f32(nk_f32_t const *a, nk_f32_t const *b, nk_size_t
     nk_each_blend_f32_neon(a, b, n, alpha, beta, r);
 #elif NK_TARGET_RVV
     nk_each_blend_f32_rvv(a, b, n, alpha, beta, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_blend_f32_v128relaxed(a, b, n, alpha, beta, r);
 #else
     nk_each_blend_f32_serial(a, b, n, alpha, beta, r);
 #endif
@@ -1621,6 +1697,8 @@ NK_PUBLIC void nk_each_blend_bf16(nk_bf16_t const *a, nk_bf16_t const *b, nk_siz
     nk_each_blend_bf16_neonbfdot(a, b, n, alpha, beta, r);
 #elif NK_TARGET_RVV
     nk_each_blend_bf16_rvv(a, b, n, alpha, beta, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_blend_bf16_v128relaxed(a, b, n, alpha, beta, r);
 #else
     nk_each_blend_bf16_serial(a, b, n, alpha, beta, r);
 #endif
@@ -1636,6 +1714,8 @@ NK_PUBLIC void nk_each_blend_f16(nk_f16_t const *a, nk_f16_t const *b, nk_size_t
     nk_each_blend_f16_neonhalf(a, b, n, alpha, beta, r);
 #elif NK_TARGET_RVV
     nk_each_blend_f16_rvv(a, b, n, alpha, beta, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_blend_f16_v128relaxed(a, b, n, alpha, beta, r);
 #else
     nk_each_blend_f16_serial(a, b, n, alpha, beta, r);
 #endif
@@ -1651,6 +1731,8 @@ NK_PUBLIC void nk_each_blend_i8(nk_i8_t const *a, nk_i8_t const *b, nk_size_t n,
     nk_each_blend_i8_neonhalf(a, b, n, alpha, beta, r);
 #elif NK_TARGET_RVV
     nk_each_blend_i8_rvv(a, b, n, alpha, beta, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_blend_i8_v128relaxed(a, b, n, alpha, beta, r);
 #else
     nk_each_blend_i8_serial(a, b, n, alpha, beta, r);
 #endif
@@ -1666,6 +1748,8 @@ NK_PUBLIC void nk_each_blend_u8(nk_u8_t const *a, nk_u8_t const *b, nk_size_t n,
     nk_each_blend_u8_neonhalf(a, b, n, alpha, beta, r);
 #elif NK_TARGET_RVV
     nk_each_blend_u8_rvv(a, b, n, alpha, beta, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_blend_u8_v128relaxed(a, b, n, alpha, beta, r);
 #else
     nk_each_blend_u8_serial(a, b, n, alpha, beta, r);
 #endif
@@ -1726,6 +1810,8 @@ NK_PUBLIC void nk_each_fma_f32(nk_f32_t const *a, nk_f32_t const *b, nk_f32_t co
     nk_each_fma_f32_neon(a, b, c, n, alpha, beta, r);
 #elif NK_TARGET_RVV
     nk_each_fma_f32_rvv(a, b, c, n, alpha, beta, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_fma_f32_v128relaxed(a, b, c, n, alpha, beta, r);
 #else
     nk_each_fma_f32_serial(a, b, c, n, alpha, beta, r);
 #endif
@@ -1741,6 +1827,8 @@ NK_PUBLIC void nk_each_fma_bf16(nk_bf16_t const *a, nk_bf16_t const *b, nk_bf16_
     nk_each_fma_bf16_neonbfdot(a, b, c, n, alpha, beta, r);
 #elif NK_TARGET_RVV
     nk_each_fma_bf16_rvv(a, b, c, n, alpha, beta, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_fma_bf16_v128relaxed(a, b, c, n, alpha, beta, r);
 #else
     nk_each_fma_bf16_serial(a, b, c, n, alpha, beta, r);
 #endif
@@ -1756,6 +1844,8 @@ NK_PUBLIC void nk_each_fma_f16(nk_f16_t const *a, nk_f16_t const *b, nk_f16_t co
     nk_each_fma_f16_neonhalf(a, b, c, n, alpha, beta, r);
 #elif NK_TARGET_RVV
     nk_each_fma_f16_rvv(a, b, c, n, alpha, beta, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_fma_f16_v128relaxed(a, b, c, n, alpha, beta, r);
 #else
     nk_each_fma_f16_serial(a, b, c, n, alpha, beta, r);
 #endif
@@ -1763,16 +1853,14 @@ NK_PUBLIC void nk_each_fma_f16(nk_f16_t const *a, nk_f16_t const *b, nk_f16_t co
 
 NK_PUBLIC void nk_each_fma_i8(nk_i8_t const *a, nk_i8_t const *b, nk_i8_t const *c, nk_size_t n, nk_f32_t const *alpha,
                               nk_f32_t const *beta, nk_i8_t *r) {
-#if NK_TARGET_SAPPHIRE
-    nk_each_fma_i8_sapphire(a, b, c, n, alpha, beta, r);
-#elif NK_TARGET_SKYLAKE
+#if NK_TARGET_SKYLAKE
     nk_each_fma_i8_skylake(a, b, c, n, alpha, beta, r);
 #elif NK_TARGET_HASWELL
     nk_each_fma_i8_haswell(a, b, c, n, alpha, beta, r);
-#elif NK_TARGET_NEONHALF
-    nk_each_fma_i8_neonhalf(a, b, c, n, alpha, beta, r);
 #elif NK_TARGET_RVV
     nk_each_fma_i8_rvv(a, b, c, n, alpha, beta, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_fma_i8_v128relaxed(a, b, c, n, alpha, beta, r);
 #else
     nk_each_fma_i8_serial(a, b, c, n, alpha, beta, r);
 #endif
@@ -1780,16 +1868,14 @@ NK_PUBLIC void nk_each_fma_i8(nk_i8_t const *a, nk_i8_t const *b, nk_i8_t const
 
 NK_PUBLIC void nk_each_fma_u8(nk_u8_t const *a, nk_u8_t const *b, nk_u8_t const *c, nk_size_t n, nk_f32_t const *alpha,
                               nk_f32_t const *beta, nk_u8_t *r) {
-#if NK_TARGET_SAPPHIRE
-    nk_each_fma_u8_sapphire(a, b, c, n, alpha, beta, r);
-#elif NK_TARGET_SKYLAKE
+#if NK_TARGET_SKYLAKE
     nk_each_fma_u8_skylake(a, b, c, n, alpha, beta, r);
 #elif NK_TARGET_HASWELL
     nk_each_fma_u8_haswell(a, b, c, n, alpha, beta, r);
-#elif NK_TARGET_NEONHALF
-    nk_each_fma_u8_neonhalf(a, b, c, n, alpha, beta, r);
 #elif NK_TARGET_RVV
     nk_each_fma_u8_rvv(a, b, c, n, alpha, beta, r);
+#elif NK_TARGET_V128RELAXED
+    nk_each_fma_u8_v128relaxed(a, b, c, n, alpha, beta, r);
 #else
     nk_each_fma_u8_serial(a, b, c, n, alpha, beta, r);
 #endif

package/include/numkong/each.hpp

@@ -199,7 +199,7 @@ void fma(in_type_ const *a, in_type_ const *b, std::size_t d, in_type_ const *c,
 
 namespace ashvardanian::numkong {
 
-#pragma region - Tensor Elementwise
+#pragma region Tensor Elementwise
 
 /** @brief Scale: output[i] = α × input[i] + β. */
 template <numeric_dtype value_type_, std::size_t max_rank_ = 8>
@@ -427,7 +427,7 @@ tensor<value_type_, allocator_type_, max_rank_> try_mul(tensor_view<value_type_,
     return result;
 }
 
-#pragma endregion - Tensor Elementwise
+#pragma endregion Tensor Elementwise
 
 } // namespace ashvardanian::numkong
 

package/include/numkong/geospatial/README.md

@@ -5,17 +5,17 @@ Both operate on arrays of latitude/longitude pairs in radians and produce distan
 
 The Haversine formula computes the great-circle distance between two points:
 
-```math
+$$
 \text{haversine}(\phi_1, \lambda_1, \phi_2, \lambda_2) = 2R \arcsin\sqrt{\sin^2\frac{\phi_2 - \phi_1}{2} + \cos\phi_1 \cos\phi_2 \sin^2\frac{\lambda_2 - \lambda_1}{2}}
-```
+$$
 
 where $R$ is Earth's mean radius and $(\phi, \lambda)$ are latitude and longitude in radians.
 
 Vincenty's formula solves the inverse geodesic problem on an oblate spheroid, iteratively refining the reduced latitude difference until convergence:
 
-```math
+$$
 \text{vincenty}(\phi_1, \lambda_1, \phi_2, \lambda_2) = b \cdot A \cdot (\sigma - \Delta\sigma)
-```
+$$
 
 where $a$ and $b$ are the equatorial and polar semi-axes of the WGS-84 ellipsoid, $\sigma$ is the angular separation, and $\Delta\sigma$ is the correction term computed through iterative convergence.
 
@@ -56,12 +56,12 @@ Each SIMD lane may converge at a different iteration count, so the kernel accumu
 Early exit uses `_mm256_movemask_pd` — when all 4 bits (for `f64`) or 8 bits (for `f32`) are set, the loop breaks.
 Coincident points and equatorial edge cases are handled by blending safe values (ones) into the intermediate terms to avoid division by zero, without requiring branches that would diverge across SIMD lanes.
 
-### Haversine Without Final Arc Conversion
+### Potential Optimization: Haversine Without Final Arc Conversion
 
-`nk_haversine_f32_haswell`, `nk_haversine_f64_haswell` support a similarity mode where the haversine formula involves $2R \cdot \text{asin}(\sqrt{h})$ and the intermediate value $h = \sin^2(\Delta\phi/2) + \cos\phi_1 \cos\phi_2 \cdot \sin^2(\Delta\lambda/2)$ is monotonic with distance.
-For ranking and comparison use cases, comparing $h$ values directly produces the same ordering as comparing full Haversine distances, since both asin and sqrt are monotonically increasing.
-This eliminates the two most expensive operations in the pipeline.
-The kernels compute the full distance by default, but the streaming API can optionally skip the final conversion when only relative ordering is needed.
+The haversine formula computes $d = 2R \cdot \text{asin}(\sqrt{h})$ where $h = \sin^2(\Delta\phi/2) + \cos\phi_1 \cos\phi_2 \cdot \sin^2(\Delta\lambda/2)$.
+Since both `asin` and `sqrt` are monotonically increasing, comparing $h$ values directly produces the same ordering as comparing full haversine distances.
+For ranking-only use cases, a future "similarity mode" could skip the final `sqrt`/`atan2` conversion and return $h$ directly, eliminating the two most expensive operations in the pipeline.
+Currently, all kernels compute the full distance.
 
 ## Performance
 
@@ -131,17 +131,17 @@ Measured with Wasmtime v42 (Cranelift backend).
 
 #### WASM
 
-Measured with Wasmtime v42 (Cranelift backend).
+Measured with Wasmtime v43 (Cranelift backend).
 
 | Kernel                         |                      ≤1° |                     ≤30° |                    ≤180° |
 | :----------------------------- | -----------------------: | -----------------------: | -----------------------: |
 | __f64__                        | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_haversine_f64_serial`      |       1.84 mp/s, 1.12 km |       1.83 mp/s, 32.8 km |        1.99 mp/s, 148 km |
-| `nk_vincenty_f64_serial`       |      0.481 mp/s, 1.86 nm |      0.419 mp/s, 2.33 nm |       0.422 mp/s, 594 nm |
-| `nk_haversine_f64_v128relaxed` |       35.7 mp/s, 1.12 km |       35.9 mp/s, 32.8 km |        35.9 mp/s, 148 km |
-| `nk_vincenty_f64_v128relaxed`  |       4.19 mp/s, 1.89 nm |       3.57 mp/s, 2.33 nm |        2.94 mp/s, 594 nm |
+| `nk_haversine_f64_serial`      |       5.12 mp/s, 1.12 km |       5.12 mp/s, 32.8 km |        5.07 mp/s, 148 km |
+| `nk_vincenty_f64_serial`       |       1.27 mp/s, 1.86 nm |       1.11 mp/s, 2.33 nm |        1.02 mp/s, 594 nm |
+| `nk_haversine_f64_v128relaxed` |        109 mp/s, 1.12 km |        109 mp/s, 32.8 km |         109 mp/s, 148 km |
+| `nk_vincenty_f64_v128relaxed`  |       12.8 mp/s, 1.89 nm |       10.4 mp/s, 2.33 nm |        8.28 mp/s, 594 nm |
 | __f32__                        | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ | ░░░░░░░░░░░░░░░░░░░░░░░░ |
-| `nk_haversine_f32_serial`      |     6.74 mp/s, 20,000 km |       6.80 mp/s, 32.7 km |        7.34 mp/s, 136 km |
-| `nk_vincenty_f32_serial`       |     2.25 mp/s, 20,000 km |        1.65 mp/s, 12.0 m |        1.35 mp/s, 22.0 m |
-| `nk_haversine_f32_v128relaxed` |      161 mp/s, 20,000 km |        165 mp/s, 32.7 km |         165 mp/s, 153 km |
-| `nk_vincenty_f32_v128relaxed`  |        24.6 mp/s, 12.0 m |        20.5 mp/s, 16.2 m |        9.57 mp/s, 18.0 m |
+| `nk_haversine_f32_serial`      |     20.9 mp/s, 20,000 km |       20.9 mp/s, 32.7 km |        21.0 mp/s, 136 km |
+| `nk_vincenty_f32_serial`       |     6.11 mp/s, 20,000 km |        4.30 mp/s, 12.0 m |        3.27 mp/s, 22.0 m |
+| `nk_haversine_f32_v128relaxed` |      523 mp/s, 20,000 km |        524 mp/s, 32.7 km |         524 mp/s, 153 km |
+| `nk_vincenty_f32_v128relaxed`  |        76.9 mp/s, 12.0 m |        68.2 mp/s, 16.2 m |        26.8 mp/s, 18.0 m |